diff --git a/flashinfer/experimental/cake/cake_kda_train/build.json b/flashinfer/experimental/cake/cake_kda_train/build.json index 55bb0c17ac6..985c627edc6 100644 --- a/flashinfer/experimental/cake/cake_kda_train/build.json +++ b/flashinfer/experimental/cake/cake_kda_train/build.json @@ -55,9 +55,9 @@ } }, "generator": { - "cake_revision": "0fb6c5f210b03ef4b08f171e574851ed6948c802", + "cake_revision": "b20c4b20c7eaf5ca5610279585cc850541151a70", "cuda": "13.4.92", - "rendered_at": "2026-11-03T20:20:58Z" + "rendered_at": "2026-11-05T11:26:08Z" }, "libraries": [ { @@ -65,7 +65,7 @@ "closure_sha256": "fb6d85f446aab7615d68497e9cd913d3359fd69a7aa3e675f565e4971eb6bd0f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_036e1c7e1dfc95a88aae_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -128,7 +128,7 @@ "closure_sha256": "712d269fc7c10b333f398e75c4f90da33c474fc67e03bf6ad230e3d428303dc4", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_058a68a5b4d89c53be4b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -443,7 +443,7 @@ "closure_sha256": "6d77155481289eb946b62d56835ba8c6f3a38dd9c74a3cd309ac00faf3f53586", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_1563b1b6e66464bbfa33_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -779,7 +779,7 @@ "closure_sha256": "f3412fb76bc083c747fafef128c40b8e489d6d00d470316b48195301d9c005f7", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_1f9c46f98f4290ee26df_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -905,7 +905,7 @@ "closure_sha256": "bbf1df4e57974fd42c36cb8c50e9dc652c61757478ef61bdde58f352c61d1719", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_22812d87c26b53a1ba10_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -968,7 +968,7 @@ "closure_sha256": "1b9bcd117cb8abc20ecb45593f227b2feb0663130135b74d9e9e72efa7d53f6b", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_22a2f04d41df18554b84_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1031,7 +1031,7 @@ "closure_sha256": "24cbff914e1bd50ee1d302a8dee09f21a3f63d8df86c0a5ad244ca7675af2dca", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_243fb0b23aea1b296ab6_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1280,7 +1280,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "99462557e95324559ff8153af5f9bbd80b58e6e5671322fa6a2e9eb9df3e9529", + "closure_sha256": "2da34ebc77cbd5f81b4ddf2355ee8c5b7b45604cc45c65be7eab1f30661c2d8f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1292,7 +1292,7 @@ "path": "aot/cake_kda_train_31fa936f5468b5749a33_sm_100a.so", "sha256": { "csrc/cake_kda_train_31fa936f5468b5749a33_binding.cu": "acd15fe5c7bb7d68beae413a56148318728653988d6f38bdd6bed5a0ef5a1770", - "csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu": "b69b68c2545e612d5528dafeddea86b987d60cdda86b552d854ec2309719820a" + "csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu": "56730d7407f76b2987a440d706ad853711f6d5b9951f6ede099ccc174f0acc1f" }, "sources": [ "csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu", @@ -1304,7 +1304,7 @@ "closure_sha256": "c918af9219b8b751a61a7c1ee66beba3585438ef393104384c42f4ebe1e423b4", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_3712054f19d4180caa28_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1451,7 +1451,7 @@ "closure_sha256": "3d3e8c8024ab208ab49493993bec65024882d3bfb7eac1fd153cb278721116a6", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_3f4b037a0a1f8dcc314b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1577,7 +1577,7 @@ "closure_sha256": "9e6dd40bd43b36d8c4dbfd5623ab6f8691cb92d1bc4ae9e4cc38e4d195c6c3bf", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_4734a5d4f63d815ec086_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1724,7 +1724,7 @@ "closure_sha256": "616262ba6230d246892ec9202f4dc573254c27226aca5c2d5cee736f5384891d", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_4cce3cc5bc6210c95ce3_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1784,7 +1784,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "ee3919821c64cbbeb3cbd4c8ea6867cfb56af94836af5902624ab51cdd2a35ab", + "closure_sha256": "3429c354d069f942de6181c72a18b28394418e455052ffbf268a849329b32287", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1796,7 +1796,7 @@ "path": "aot/cake_kda_train_54786ffd01bd9b6bb416_sm_100a.so", "sha256": { "csrc/cake_kda_train_54786ffd01bd9b6bb416_binding.cu": "a77d517dc4c55ad6ddcf1797a307fb02d0b67ba8ecdf3d31ca2fd7e7a2de000d", - "csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu": "584354d5bce463661438e5324be10347b08d50037ce3b42feb7d290f30f2a96a" + "csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu": "c5ae178a848262845224ab630925a8aa7fe4d73a223c02eea4100cc5cb179ef5" }, "sources": [ "csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu", @@ -1871,7 +1871,7 @@ "closure_sha256": "e56687506bb4a8257d2233fa90850f924b76ca7a5713bae6c85496ed017547d5", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_5f5a5b3715675e083823_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1934,7 +1934,7 @@ "closure_sha256": "31dffe350bac3b135b79496a18fe8ff54fd2f694c13b799b6a44548ac34fe645", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_5fcc76904e5e2dd891de_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2060,7 +2060,7 @@ "closure_sha256": "1143d0e7358c89edf4c4a04a7bc8e78432e6e7d09e7a185a5e1c7e9dbd850128", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_706240be56d4e41df1a2_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2186,7 +2186,7 @@ "closure_sha256": "b4e1c6c27b131d624b5523fe1bfba006afcfb32ab55eb51a88d965aa69a5aa5d", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_7eca60b36e233943dae6_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2372,7 +2372,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "997544e8154ec636d2523d6b93e8621d880a66b4fe38fd5e42dda40162afc702", + "closure_sha256": "29157dbfd3026aabb88e5598581e3286c2b09a2d35645d5083853d7966ea942b", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -2384,7 +2384,7 @@ "path": "aot/cake_kda_train_87b73f813c841159bf43_sm_100a.so", "sha256": { "csrc/cake_kda_train_87b73f813c841159bf43_binding.cu": "c0da28f7fa1049348c725e78a73f3b6dcdd10868184119a7f079f092cf315179", - "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu": "7ba3a424d66ed5ebbcc10f87d39dd83914a9ed6dd3f36dcbf4f72d5e1910f4fe" + "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu": "e22ac309ae36b47824d406eef60729bcc5abd6dd9ccd2db87904005664681133" }, "sources": [ "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu", @@ -2393,7 +2393,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "ea11b0bd27384f6857d205221258ec2947c158809aa114cb5313463ee79b99b3", + "closure_sha256": "be788de94c04b04343a128a62c63e97c0000737e0226cefb40e53cea054027bc", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -2405,7 +2405,7 @@ "path": "aot/cake_kda_train_87b73f813c841159bf43_sm_103a.so", "sha256": { "csrc/cake_kda_train_87b73f813c841159bf43_binding.cu": "c0da28f7fa1049348c725e78a73f3b6dcdd10868184119a7f079f092cf315179", - "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu": "7ba3a424d66ed5ebbcc10f87d39dd83914a9ed6dd3f36dcbf4f72d5e1910f4fe" + "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu": "e22ac309ae36b47824d406eef60729bcc5abd6dd9ccd2db87904005664681133" }, "sources": [ "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu", @@ -2417,7 +2417,7 @@ "closure_sha256": "786c43a57719359c1beb01485fb1d25442f9cfb4c4903688b7066f7835fc117f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_898a1eb972fbaf55789a_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2480,7 +2480,7 @@ "closure_sha256": "e6e930a4181c41abd6c0457eb1b531c7d8b63e39add6ff252b458a3bddb71908", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_8ab0929b8154c90b5eff_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2540,7 +2540,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "c9c2d8bd31381f2f1bd5546e1ed0e918382320dfa000480bfe7a2c79a3ecc604", + "closure_sha256": "1c7d14244747d80799cd40a99adec88417ae2a52fed7a95b0de30ea5c3ba7e1b", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -2552,7 +2552,7 @@ "path": "aot/cake_kda_train_8ce96eeb5b7517229400_sm_107a.so", "sha256": { "csrc/cake_kda_train_8ce96eeb5b7517229400_binding.cu": "3d3dbf0da8bfa77b603066798ee00288f4997c1d72cec5ff09083acc9bd546ec", - "csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu": "2257d9d1d6412d8c601b15c44e9a2493e4e0e64810e60d2803004ba712a2e776" + "csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu": "58e18a7c130094c763b82c97b6e9a7c20153afe9265e3b9aef41854ec336c4b6" }, "sources": [ "csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu", @@ -2690,7 +2690,7 @@ "closure_sha256": "70e36905352ceea9458e0c5b08237451325c082180dbf17c8c536394c0505343", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_8f3f4fb30feec66092de_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2816,7 +2816,7 @@ "closure_sha256": "f8e6586074e0ba7cc1d36c1c5d5e5df11db9060580bbd1c448ed9e52d2ecbc11", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_909a0aa57efb3f353972_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2942,7 +2942,7 @@ "closure_sha256": "d8ebfc1b6c7b3f6dda89a22b236fe6b80a2d626c1691c8d0b7d8dc05d1d35dbb", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_945a87fa84a265fc9f94_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3005,7 +3005,7 @@ "closure_sha256": "a285ec2632103aec9553371ef3384df0e7aa2d7e443e1863bfba3af63d5b1434", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_94dcf92dabd8cc421b8f_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3152,7 +3152,7 @@ "closure_sha256": "0912ef8a3f3eefd6fcf2b9a22d00534cba2495f4543249c15164c69eda545ccd", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_9931cdff5dfd86c687c7_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3215,7 +3215,7 @@ "closure_sha256": "4e72179e078746bee49d2a82cc8e9fc1ec4c536ca912d8e20ee729e602c34c4e", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_a557058a8aeca49a6402_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3275,7 +3275,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "c3cbf2d5315b8dd39f4ed945a012e5b7065862b5340f8a9336a9910005d37ff8", + "closure_sha256": "177c731ea041e17af99bfcd7e4de395cfb60fe43df8281974c7a991912975fff", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -3287,7 +3287,7 @@ "path": "aot/cake_kda_train_aa6b5c23c97e1cb9880a_sm_107a.so", "sha256": { "csrc/cake_kda_train_aa6b5c23c97e1cb9880a_binding.cu": "92b299e2442b425315c2a48d87fc8715b414a723516f8155842a95263970385a", - "csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu": "c0b2637c19e401d19bb4c568bb42cdf0d1063272d50cf42f1b8a2a8c685627ac" + "csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu": "c85805e47b1ff2edba5f87fad10e5fc7c8f47ce243a9fc0b665e190d6990b51e" }, "sources": [ "csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu", @@ -3299,7 +3299,7 @@ "closure_sha256": "b07f5f119b0f0c671e1c9f130dddb4d6171b9fe7ea4886ba0a845f20c6078b21", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_ac974e171d56fc3ce10b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3611,7 +3611,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "85d2c2aed90d5bca7837f738fc6aa255fe7dfb7d46aa052925b13a378d3483f0", + "closure_sha256": "736611400f691d1e6b9d21b1a4de5b87d185faf644e7d799db6a0ed89c078964", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -3623,7 +3623,7 @@ "path": "aot/cake_kda_train_c27b09eb5f5881be2969_sm_103a.so", "sha256": { "csrc/cake_kda_train_c27b09eb5f5881be2969_binding.cu": "72b08e731d61855c9b753d98f4f3303dc2f4cd5efbac1ed1be0443f75da02b05", - "csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu": "37a294a92553fd7497de5fafbcf6b593aeb0af772dcb066570fdb10eb3724553" + "csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu": "eb973be56b0c1e6834d5bb0a0a76394cb04e50b5810e6474f4436202cc79fbb4" }, "sources": [ "csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu", @@ -3782,7 +3782,7 @@ "closure_sha256": "d14da6eef2ff3e91ddf879047dd08ff5aad9c072b3c530e071002b4f41491ac9", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_d32455627b87ba0a1d87_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3842,7 +3842,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "a145e2981e4b53d084f181e059954472d86c3f7c9472dc592ec3af444acebb4e", + "closure_sha256": "66ed4fe1e61052a772c3cb4dea04cf1f01a79c2b3d8255c2f9e29166334ce700", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -3854,7 +3854,7 @@ "path": "aot/cake_kda_train_d5fd3f7234a4266e09d2_sm_100a.so", "sha256": { "csrc/cake_kda_train_d5fd3f7234a4266e09d2_binding.cu": "d64b14a931f8f57852c590ce0e8aa3c671dfd6b8ab718c53a4a6c98873dd8fb3", - "csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu": "2c6f0812dbd5db0216cb32155250fe6214e2273455c6254259b4b1610228f608" + "csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu": "92a99290857d4395558ddacca22dbf567907b2052528cf071a55b1925ede755a" }, "sources": [ "csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu", @@ -4013,7 +4013,7 @@ "closure_sha256": "c6d5dd2df0f40e42e48a1d562ceba551a678682d23f8fab8c5d0c73021fd0e4b", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_dec885b973bd44f88416_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4265,7 +4265,7 @@ "closure_sha256": "18945915ee030fd9a795214e8ddfe221ee8576f3cbe431ff79e7f965a14f8845", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_e60e7ca5d88192193e61_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4325,7 +4325,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "20d25a83809d70c90373aa7bcec3cee701047767d6748753f9e47d6ab99b8c32", + "closure_sha256": "db14d39a4cbab2729c1464fc1aa1d14459ffc70e06d169d92e8cb2e4337c46af", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4337,7 +4337,7 @@ "path": "aot/cake_kda_train_ea14e16d1fc71364ba03_sm_107a.so", "sha256": { "csrc/cake_kda_train_ea14e16d1fc71364ba03_binding.cu": "5f19248aea28a42fb7c6e76ab607844338ea1e51c9f61b392dc47ed6a3d63c54", - "csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu": "1f507cffec0c6be6a1d33573c5e0d56ac14ea13fcbc63195e72373e9652355fe" + "csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu": "d97b83f275a6f9d6fd8bd229923f8c4e5c19e55a55c075cd7e7d1489760f5140" }, "sources": [ "csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu", @@ -4412,7 +4412,7 @@ "closure_sha256": "a674e25813771bcca4f3c050376cc2025ef0ac750fbe380faac63b461b6f9be5", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_eac26fffb3df8780302c_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4517,7 +4517,7 @@ "closure_sha256": "a2cf1b4d76a4e690725a5dbd3634cdce9600d9181866ab2a351ea0c8c26569b2", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_f011cd9add8f3c7c3eee_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4640,7 +4640,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "012b974c3ab344ff46c357afbc659b26eb5d7d529f403355cec7ed50f914933b", + "closure_sha256": "0258bf6cf18edf2be46cbf176a04c9a773eac1bf6791b21e9089b0c05e65b46a", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -4652,7 +4652,7 @@ "path": "aot/cake_kda_train_f4ea302627f05d805437_sm_103a.so", "sha256": { "csrc/cake_kda_train_f4ea302627f05d805437_binding.cu": "92110639aeb17b136f85cf4fab2265c3a7efa549bd26b1aa142cd7f0cb28b0f0", - "csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu": "9b8a30bb07e1f13d83e7ddd87912691b8be4138fa51b0e7d99376b2ddebd912f" + "csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu": "9f0f17ebc89518adf4ed026985554ee0e458ef409907e1a2694b12b333225ed3" }, "sources": [ "csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu", @@ -4664,7 +4664,7 @@ "closure_sha256": "6282925a4476ddb4cb2d5f06faca55010bcd672651866c159766d23e0b84ad97", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_f6a70fb0288a508f2828_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4727,7 +4727,7 @@ "closure_sha256": "5af2fdaec13f101c0636be4d72f29563d75593a3fe25587edf7994fec1b19779", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_f92b256cccc2c9007554_sm_100a", "nvcc_flags": [ "--use_fast_math" diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu index f22e7eab9d7..a5fba7adf29 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -199,6 +200,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2267,12 +2284,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_4); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_40); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_40, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_40 : __fmaf_rn(dl, _rcp_170, e2_40)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2296,12 +2319,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_5); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_41); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_41, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_41, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_41 : __fmaf_rn(dl_1, _rcp_174, e2_41)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2325,12 +2354,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_6); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_42); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_42, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_42, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_42 : __fmaf_rn(dl_3, _rcp_178, e2_42)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2354,12 +2389,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_7); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_43); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_43, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_43, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_43 : __fmaf_rn(dl_5, _rcp_182, e2_43)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2395,12 +2436,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2420,12 +2467,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2445,12 +2498,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2470,12 +2529,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2523,12 +2588,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_0 = sub_f32x2(_f2_64, _f2_8); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2552,12 +2623,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_1 = sub_f32x2(_f2_64, _f2_9); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2581,12 +2658,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_2 = sub_f32x2(_f2_64, _f2_10); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2610,12 +2693,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_3 = sub_f32x2(_f2_64, _f2_11); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2657,12 +2746,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_4 = sub_f32x2(_f2_70, _f2_12); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2686,12 +2781,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_5 = sub_f32x2(_f2_70, _f2_13); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2715,12 +2816,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_6 = sub_f32x2(_f2_70, _f2_14); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2744,12 +2851,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_7 = sub_f32x2(_f2_70, _f2_15); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2820,37 +2933,37 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_60.x); float e1_12 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_61 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_61.x); float e1_13 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_62 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_62.x); float e1_14 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_63 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_63.x); float e1_15 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2862,29 +2975,29 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2892,29 +3005,29 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2940,402 +3053,133 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int sb8 = 32 + 8 * th; int gb8 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8 * 128); int kb8 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8 * 128); - float dbp[8]; - float qo[8]; - tmem_ld_x8(&qo[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float ko[8]; - tmem_ld_x8(&ko[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float qdg[8]; - tmem_ld_x8(&qdg[0], taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)); - float kdg[8]; - tmem_ld_x8(&kdg[0], taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th)); - float dkt_0[8]; - tmem_ld_x8(&dkt_0[0], taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th)); - float btv[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[0])), "=r"(*reinterpret_cast(&btv[(0) + 1])), "=r"(*reinterpret_cast(&btv[(0) + 2])), "=r"(*reinterpret_cast(&btv[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[4])), "=r"(*reinterpret_cast(&btv[(4) + 1])), "=r"(*reinterpret_cast(&btv[(4) + 2])), "=r"(*reinterpret_cast(&btv[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int c_bp = 144 + 8 * th; float s_run[1]; s_run[0] = 0.0f; - int t_0 = 32 + 8 * th + 7; - float r_248[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_248[0]; - unsigned int r_249[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_249[0] << 16); - unsigned int r_250[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_250[0] << 16); - float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew + 896 + d_lane]; - float dkf_0 = pool_f32[ew + 1920 + d_lane]; - float dgf_0 = pool_f32[ew + 2944 + d_lane]; - float bt_t_0 = btv[7]; - float e_off_0 = approx_exp2(g_t_0 - gn); - float e_dg_0 = approx_exp2(g_t_0 - gc_0); - float dq2_0 = __fmaf_rn(qo[7], e_off_0, qdg[7] * e_dg_0); - float dk2_0 = __fmaf_rn(ko[7], e_off_0, kdg[7] * e_dg_0); - dbp[7] = dk2_0 * k_t_0; - float dk2b_0 = dk2_0 * bt_t_0; - float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + dkt_0[7]; - float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, (dk2b_0 - dkt_0[7]) * k_t_0) + dgf_0; - s_run[0] = s_run[0] + dg_o_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 7)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_0 = t_0 * qrs; - if (t_0 < clen) { - { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; - } - { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; - } - } - int t_1 = 32 + 8 * th + 6; - float r_251[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_251[0]; - unsigned int r_252[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((kb8 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_252[0] << 16); - unsigned int r_253[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_253[0] << 16); - float q_t_1_1 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew + 768 + d_lane]; - float dkf_1 = pool_f32[ew + 1792 + d_lane]; - float dgf_1 = pool_f32[ew + 2816 + d_lane]; - float bt_t_1 = btv[6]; - float e_off_1 = approx_exp2(g_t_1 - gn); - float e_dg_1 = approx_exp2(g_t_1 - gc_0); - float dq2_1 = __fmaf_rn(qo[6], e_off_1, qdg[6] * e_dg_1); - float dk2_1 = __fmaf_rn(ko[6], e_off_1, kdg[6] * e_dg_1); - dbp[6] = dk2_1 * k_t_1; - float dk2b_1 = dk2_1 * bt_t_1; - float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + dkt_0[6]; - float dg_o_1 = __fmaf_rn(q_t_1_1, dq2_1, (dk2b_1 - dkt_0[6]) * k_t_1) + dgf_1; - s_run[0] = s_run[0] + dg_o_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 6)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1 = t_1 * qrs; - if (t_1 < clen) { - { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; - } - { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; - } - } - int t_2 = 32 + 8 * th + 5; - float r_254[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_254[0]; - unsigned int r_255[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((kb8 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_255[0] << 16); - unsigned int r_256[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_256[0] << 16); - float q_t_1_2 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew + 640 + d_lane]; - float dkf_2 = pool_f32[ew + 1664 + d_lane]; - float dgf_2 = pool_f32[ew + 2688 + d_lane]; - float bt_t_2 = btv[5]; - float e_off_2 = approx_exp2(g_t_2 - gn); - float e_dg_2 = approx_exp2(g_t_2 - gc_0); - float dq2_2 = __fmaf_rn(qo[5], e_off_2, qdg[5] * e_dg_2); - float dk2_2 = __fmaf_rn(ko[5], e_off_2, kdg[5] * e_dg_2); - dbp[5] = dk2_2 * k_t_2; - float dk2b_2 = dk2_2 * bt_t_2; - float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + dkt_0[5]; - float dg_o_2 = __fmaf_rn(q_t_1_2, dq2_2, (dk2b_2 - dkt_0[5]) * k_t_2) + dgf_2; - s_run[0] = s_run[0] + dg_o_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 5)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_2 = t_2 * qrs; - if (t_2 < clen) { - { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; - } - { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; - } - } - int t_3 = 32 + 8 * th + 4; - float r_257[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_257[0]; - unsigned int r_258[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((kb8 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_258[0] << 16); - unsigned int r_259[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_259[0] << 16); - float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew + 512 + d_lane]; - float dkf_3 = pool_f32[ew + 1536 + d_lane]; - float dgf_3 = pool_f32[ew + 2560 + d_lane]; - float bt_t_3 = btv[4]; - float e_off_3 = approx_exp2(g_t_3 - gn); - float e_dg_3 = approx_exp2(g_t_3 - gc_0); - float dq2_3 = __fmaf_rn(qo[4], e_off_3, qdg[4] * e_dg_3); - float dk2_3 = __fmaf_rn(ko[4], e_off_3, kdg[4] * e_dg_3); - dbp[4] = dk2_3 * k_t_3; - float dk2b_3 = dk2_3 * bt_t_3; - float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + dkt_0[4]; - float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, (dk2b_3 - dkt_0[4]) * k_t_3) + dgf_3; - s_run[0] = s_run[0] + dg_o_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 4)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_3 = t_3 * qrs; - if (t_3 < clen) { - { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; - } - { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; - } - } - int t_4 = 32 + 8 * th + 3; - float r_260[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_260[0]; - unsigned int r_261[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((kb8 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_261[0] << 16); - unsigned int r_262[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_262[0] << 16); - float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew + 384 + d_lane]; - float dkf_4 = pool_f32[ew + 1408 + d_lane]; - float dgf_4 = pool_f32[ew + 2432 + d_lane]; - float bt_t_4 = btv[3]; - float e_off_4 = approx_exp2(g_t_4 - gn); - float e_dg_4 = approx_exp2(g_t_4 - gc_0); - float dq2_4 = __fmaf_rn(qo[3], e_off_4, qdg[3] * e_dg_4); - float dk2_4 = __fmaf_rn(ko[3], e_off_4, kdg[3] * e_dg_4); - dbp[3] = dk2_4 * k_t_4; - float dk2b_4 = dk2_4 * bt_t_4; - float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + dkt_0[3]; - float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, (dk2b_4 - dkt_0[3]) * k_t_4) + dgf_4; - s_run[0] = s_run[0] + dg_o_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 3)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_4 = t_4 * qrs; - if (t_4 < clen) { - { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; - } - { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; - } - } - int t_5 = 32 + 8 * th + 2; - float r_263[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_263[0]; - unsigned int r_264[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((kb8 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_264[0] << 16); - unsigned int r_265[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_265[0] << 16); - float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew + 256 + d_lane]; - float dkf_5 = pool_f32[ew + 1280 + d_lane]; - float dgf_5 = pool_f32[ew + 2304 + d_lane]; - float bt_t_5 = btv[2]; - float e_off_5 = approx_exp2(g_t_5 - gn); - float e_dg_5 = approx_exp2(g_t_5 - gc_0); - float dq2_5 = __fmaf_rn(qo[2], e_off_5, qdg[2] * e_dg_5); - float dk2_5 = __fmaf_rn(ko[2], e_off_5, kdg[2] * e_dg_5); - dbp[2] = dk2_5 * k_t_5; - float dk2b_5 = dk2_5 * bt_t_5; - float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + dkt_0[2]; - float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, (dk2b_5 - dkt_0[2]) * k_t_5) + dgf_5; - s_run[0] = s_run[0] + dg_o_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 2)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_5 = t_5 * qrs; - if (t_5 < clen) { - { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; - } - { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; - } - } - int t_6 = 32 + 8 * th + 1; - float r_266[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_266[0]; - unsigned int r_267[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_267[0] << 16); - unsigned int r_268[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_268[0] << 16); - float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew + 128 + d_lane]; - float dkf_6 = pool_f32[ew + 1152 + d_lane]; - float dgf_6 = pool_f32[ew + 2176 + d_lane]; - float bt_t_6 = btv[1]; - float e_off_6 = approx_exp2(g_t_6 - gn); - float e_dg_6 = approx_exp2(g_t_6 - gc_0); - float dq2_6 = __fmaf_rn(qo[1], e_off_6, qdg[1] * e_dg_6); - float dk2_6 = __fmaf_rn(ko[1], e_off_6, kdg[1] * e_dg_6); - dbp[1] = dk2_6 * k_t_6; - float dk2b_6 = dk2_6 * bt_t_6; - float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + dkt_0[1]; - float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, (dk2b_6 - dkt_0[1]) * k_t_6) + dgf_6; - s_run[0] = s_run[0] + dg_o_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 1)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_6 = t_6 * qrs; - if (t_6 < clen) { - { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; - } - { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; - } - } - int t_7 = 32 + 8 * th; - float r_269[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_269[0]; - unsigned int r_270[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((kb8 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_270[0] << 16); - unsigned int r_271[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_271[0] << 16); - float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew + d_lane]; - float dkf_7 = pool_f32[ew + 1024 + d_lane]; - float dgf_7 = pool_f32[ew + 2048 + d_lane]; - float bt_t_7 = btv[0]; - float e_off_7 = approx_exp2(g_t_7 - gn); - float e_dg_7 = approx_exp2(g_t_7 - gc_0); - float dq2_7 = __fmaf_rn(qo[0], e_off_7, qdg[0] * e_dg_7); - float dk2_7 = __fmaf_rn(ko[0], e_off_7, kdg[0] * e_dg_7); - dbp[0] = dk2_7 * k_t_7; - float dk2b_7 = dk2_7 * bt_t_7; - float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + dkt_0[0]; - float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, (dk2b_7 - dkt_0[0]) * k_t_7) + dgf_7; - s_run[0] = s_run[0] + dg_o_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_7 = t_7 * qrs; - if (t_7 < clen) { - { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; - } - { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j = 7 - k; + float qo[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qo[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + j))); + float ko[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(ko[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th + j))); + float qdg[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qdg[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + j))); + float kdg[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(kdg[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th + j))); + float dkt_0[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(dkt_0[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th + j))); + float btv[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btv[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th + j)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8 + j; + float r_248[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_248[0]; + unsigned int r_249[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_249[0] << 16); + unsigned int r_250[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_250[0] << 16); + float q_t_1 = ((t >= clen) ? 0.0f : q_t); + float bt_t = btv[0]; + float e_off = approx_exp2(g_t - gn); + float e_dg = approx_exp2(g_t - gc_0); + float dq2 = __fmaf_rn(qo[0], e_off, qdg[0] * e_dg); + float dk2 = __fmaf_rn(ko[0], e_off, kdg[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float qe = q_t_1 * e_dg; + float kp = k_t * bt_t; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg) * kdg[0]); + float dqf = pool_f32[ew + 128 * j + d_lane]; + float dkf = pool_f32[ew + (1024 + 128 * j) + d_lane]; + float dgf = pool_f32[ew + (2048 + 128 * j) + d_lane]; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + dkt_0[0]; + float dg_o = __fmaf_rn(q_t_1, dq2, __fmaf_rn(dk2b - dkt_0[0], k_t, cr)) + dgf; + s_run[0] = s_run[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + j)), "f"(s_run[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff = t * qrs; + if (t < clen) { + { + auto _gst_248 = dq_base + qoff; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o; + } + { + auto _gst_249 = dk_base + qoff; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o; + } } } float tot = s_run[0]; + float dbp[8]; + tmem_ld_x8(&dbp[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); int b4 = lane_1 >> 4 & 1; int b3 = lane_1 >> 3 & 1; int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_250; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_251; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_252; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_253; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_254; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_255; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_256; + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_257; + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_258; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3346,9 +3190,9 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d if (lane_1 == 17) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(dt_bias + (hv * 128 + 32 * qd)))); } mbarrier_wait(epi_full_addr + (th) * 8, 1); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_272[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_0 = r_272[0]; + float r_251[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_0 = r_251[0]; float gc_1 = gn_0 - 21.640425f; int eb_0 = 98304 + th * -32768; int ew_0 = eb_0 >> 2; @@ -3356,402 +3200,133 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int sb8_0 = 48 + 8 * th; int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); - float dbp_0[8]; - float qo_0[8]; - tmem_ld_x8(&qo_0[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float ko_0[8]; - tmem_ld_x8(&ko_0[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float qdg_0[8]; - tmem_ld_x8(&qdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float kdg_0[8]; - tmem_ld_x8(&kdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float dkt_1[8]; - tmem_ld_x8(&dkt_1[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_0[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_0[0])), "=r"(*reinterpret_cast(&btv_0[(0) + 1])), "=r"(*reinterpret_cast(&btv_0[(0) + 2])), "=r"(*reinterpret_cast(&btv_0[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_0[4])), "=r"(*reinterpret_cast(&btv_0[(4) + 1])), "=r"(*reinterpret_cast(&btv_0[(4) + 2])), "=r"(*reinterpret_cast(&btv_0[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int c_bp_0 = 176 + 8 * th; float s_run_0[1]; s_run_0[0] = 0.0f; - int t_8 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_0 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_0 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_0 + 2944 + d_lane]; - float bt_t_8 = btv_0[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_0); - float e_dg_8 = approx_exp2(g_t_8 - gc_1); - float dq2_8 = __fmaf_rn(qo_0[7], e_off_8, qdg_0[7] * e_dg_8); - float dk2_8 = __fmaf_rn(ko_0[7], e_off_8, kdg_0[7] * e_dg_8); - dbp_0[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + dkt_1[7]; - float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, (dk2b_8 - dkt_1[7]) * k_t_8) + dgf_8; - s_run_0[0] = s_run_0[0] + dg_o_8; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_8 = t_8 * qrs; - if (t_8 < clen) { - { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; - } - { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; - } - } - int t_9 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_0 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_0 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_0 + 2816 + d_lane]; - float bt_t_9 = btv_0[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_0); - float e_dg_9 = approx_exp2(g_t_9 - gc_1); - float dq2_9 = __fmaf_rn(qo_0[6], e_off_9, qdg_0[6] * e_dg_9); - float dk2_9 = __fmaf_rn(ko_0[6], e_off_9, kdg_0[6] * e_dg_9); - dbp_0[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + dkt_1[6]; - float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, (dk2b_9 - dkt_1[6]) * k_t_9) + dgf_9; - s_run_0[0] = s_run_0[0] + dg_o_9; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_9 = t_9 * qrs; - if (t_9 < clen) { - { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; - } - { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; - } - } - int t_10 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_0 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_0 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_0 + 2688 + d_lane]; - float bt_t_10 = btv_0[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_0); - float e_dg_10 = approx_exp2(g_t_10 - gc_1); - float dq2_10 = __fmaf_rn(qo_0[5], e_off_10, qdg_0[5] * e_dg_10); - float dk2_10 = __fmaf_rn(ko_0[5], e_off_10, kdg_0[5] * e_dg_10); - dbp_0[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + dkt_1[5]; - float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, (dk2b_10 - dkt_1[5]) * k_t_10) + dgf_10; - s_run_0[0] = s_run_0[0] + dg_o_10; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_10 = t_10 * qrs; - if (t_10 < clen) { - { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; - } - { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; - } - } - int t_11 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_0 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_0 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_0 + 2560 + d_lane]; - float bt_t_11 = btv_0[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_0); - float e_dg_11 = approx_exp2(g_t_11 - gc_1); - float dq2_11 = __fmaf_rn(qo_0[4], e_off_11, qdg_0[4] * e_dg_11); - float dk2_11 = __fmaf_rn(ko_0[4], e_off_11, kdg_0[4] * e_dg_11); - dbp_0[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + dkt_1[4]; - float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, (dk2b_11 - dkt_1[4]) * k_t_11) + dgf_11; - s_run_0[0] = s_run_0[0] + dg_o_11; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_11 = t_11 * qrs; - if (t_11 < clen) { - { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; - } - { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; - } - } - int t_12 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_0 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_0 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_0 + 2432 + d_lane]; - float bt_t_12 = btv_0[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_0); - float e_dg_12 = approx_exp2(g_t_12 - gc_1); - float dq2_12 = __fmaf_rn(qo_0[3], e_off_12, qdg_0[3] * e_dg_12); - float dk2_12 = __fmaf_rn(ko_0[3], e_off_12, kdg_0[3] * e_dg_12); - dbp_0[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + dkt_1[3]; - float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, (dk2b_12 - dkt_1[3]) * k_t_12) + dgf_12; - s_run_0[0] = s_run_0[0] + dg_o_12; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_12 = t_12 * qrs; - if (t_12 < clen) { - { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; - } - { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; - } - } - int t_13 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_0 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_0 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_0 + 2304 + d_lane]; - float bt_t_13 = btv_0[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_0); - float e_dg_13 = approx_exp2(g_t_13 - gc_1); - float dq2_13 = __fmaf_rn(qo_0[2], e_off_13, qdg_0[2] * e_dg_13); - float dk2_13 = __fmaf_rn(ko_0[2], e_off_13, kdg_0[2] * e_dg_13); - dbp_0[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + dkt_1[2]; - float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, (dk2b_13 - dkt_1[2]) * k_t_13) + dgf_13; - s_run_0[0] = s_run_0[0] + dg_o_13; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_13 = t_13 * qrs; - if (t_13 < clen) { - { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; - } - { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; - } - } - int t_14 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_0 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_0 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_0 + 2176 + d_lane]; - float bt_t_14 = btv_0[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_0); - float e_dg_14 = approx_exp2(g_t_14 - gc_1); - float dq2_14 = __fmaf_rn(qo_0[1], e_off_14, qdg_0[1] * e_dg_14); - float dk2_14 = __fmaf_rn(ko_0[1], e_off_14, kdg_0[1] * e_dg_14); - dbp_0[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + dkt_1[1]; - float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, (dk2b_14 - dkt_1[1]) * k_t_14) + dgf_14; - s_run_0[0] = s_run_0[0] + dg_o_14; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_14 = t_14 * qrs; - if (t_14 < clen) { - { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; - } - { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; - } - } - int t_15 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_0 + d_lane]; - float dkf_15 = pool_f32[ew_0 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_0 + 2048 + d_lane]; - float bt_t_15 = btv_0[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_0); - float e_dg_15 = approx_exp2(g_t_15 - gc_1); - float dq2_15 = __fmaf_rn(qo_0[0], e_off_15, qdg_0[0] * e_dg_15); - float dk2_15 = __fmaf_rn(ko_0[0], e_off_15, kdg_0[0] * e_dg_15); - dbp_0[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + dkt_1[0]; - float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, (dk2b_15 - dkt_1[0]) * k_t_15) + dgf_15; - s_run_0[0] = s_run_0[0] + dg_o_15; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_15 = t_15 * qrs; - if (t_15 < clen) { - { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; - } - { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_1 = 7 - k_1; + float qo_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qo_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th + j_1))); + float ko_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(ko_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th + j_1))); + float qdg_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qdg_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + j_1))); + float kdg_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(kdg_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th + j_1))); + float dkt_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(dkt_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th + j_1))); + float btv_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btv_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_0 + j_1; + float r_252[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_252[0]; + unsigned int r_253[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_253[0] << 16); + unsigned int r_254[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_254[0] << 16); + float q_t_1_1 = ((t_1 >= clen) ? 0.0f : q_t_2); + float bt_t_1 = btv_1[0]; + float e_off_1 = approx_exp2(g_t_1 - gn_0); + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float dq2_1 = __fmaf_rn(qo_1[0], e_off_1, qdg_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(ko_1[0], e_off_1, kdg_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float qe_1 = q_t_1_1 * e_dg_1; + float kp_12 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_12; + float kbe_1 = (float)kb_1 * e_dg_1; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg_1[0], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_12 * e_dg_1) * kdg_1[0]); + float dqf_1 = pool_f32[ew_0 + 128 * j_1 + d_lane]; + float dkf_1 = pool_f32[ew_0 + (1024 + 128 * j_1) + d_lane]; + float dgf_1 = pool_f32[ew_0 + (2048 + 128 * j_1) + d_lane]; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + dkt_1[0]; + float dg_o_1 = __fmaf_rn(q_t_1_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_1[0], k_t_1, cr_1)) + dgf_1; + s_run_0[0] = s_run_0[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + j_1)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_1)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff_1 = t_1 * qrs; + if (t_1 < clen) { + { + auto _gst_259 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_1; + } + { + auto _gst_260 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_1; + } } } float tot_0 = s_run_0[0]; + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); int b4_0 = lane_1 >> 4 & 1; int b3_0 = lane_1 >> 3 & 1; int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_261; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_262; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_263; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_264; float v2_16[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_265; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_266; float send_12 = ((b2_0 == 0) ? v2_16[1] : v2_16[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_267; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_268; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_269; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); float gacc[8]; @@ -3803,8 +3378,8 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); + uint32_t _addr_270 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_270), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3836,132 +3411,132 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_267 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_271 = make_float2(gacc[4], gacc[4]); float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&rp_u)); - float _abs_270; - asm("abs.f32 %0, %1;" : "=f"(_abs_270) : "f"(rp_z.x)); - float _neg_269; - asm("neg.f32 %0, %1;" : "=f"(_neg_269) : "f"(_abs_270)); - float _exp_268 = __nv_expf(_neg_269); - float _abs_273; - asm("abs.f32 %0, %1;" : "=f"(_abs_273) : "f"(rp_z.y)); - float _neg_272; - asm("neg.f32 %0, %1;" : "=f"(_neg_272) : "f"(_abs_273)); - float _exp_271 = __nv_expf(_neg_272); - float2 rp_e = make_float2(_exp_268, _exp_271); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_271), "l"(*(const unsigned long long*)&rp_u)); + float _abs_274; + asm("abs.f32 %0, %1;" : "=f"(_abs_274) : "f"(rp_z.x)); + float _neg_273; + asm("neg.f32 %0, %1;" : "=f"(_neg_273) : "f"(_abs_274)); + float _exp_272 = __nv_expf(_neg_273); + float _abs_277; + asm("abs.f32 %0, %1;" : "=f"(_abs_277) : "f"(rp_z.y)); + float _neg_276; + asm("neg.f32 %0, %1;" : "=f"(_neg_276) : "f"(_abs_277)); + float _exp_275 = __nv_expf(_neg_276); + float2 rp_e = make_float2(_exp_272, _exp_275); float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_274 = approx_rcp(rp_den.x); - float _rcp_275 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_276 = make_float2(_rcp_274, _rcp_275); + float _rcp_278 = approx_rcp(rp_den.x); + float _rcp_279 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_280 = make_float2(_rcp_278, _rcp_279); float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_276)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_280)); float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_277 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_281 = make_float2(gacc[5], gacc[5]); float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_281)); u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_278 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_282 = make_float2(gacc[4], gacc[4]); float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_278), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_281; - asm("abs.f32 %0, %1;" : "=f"(_abs_281) : "f"(rp_z_0.x)); - float _neg_280; - asm("neg.f32 %0, %1;" : "=f"(_neg_280) : "f"(_abs_281)); - float _exp_279 = __nv_expf(_neg_280); - float _abs_284; - asm("abs.f32 %0, %1;" : "=f"(_abs_284) : "f"(rp_z_0.y)); - float _neg_283; - asm("neg.f32 %0, %1;" : "=f"(_neg_283) : "f"(_abs_284)); - float _exp_282 = __nv_expf(_neg_283); - float2 rp_e_0 = make_float2(_exp_279, _exp_282); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_282), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_285; + asm("abs.f32 %0, %1;" : "=f"(_abs_285) : "f"(rp_z_0.x)); + float _neg_284; + asm("neg.f32 %0, %1;" : "=f"(_neg_284) : "f"(_abs_285)); + float _exp_283 = __nv_expf(_neg_284); + float _abs_288; + asm("abs.f32 %0, %1;" : "=f"(_abs_288) : "f"(rp_z_0.y)); + float _neg_287; + asm("neg.f32 %0, %1;" : "=f"(_neg_287) : "f"(_abs_288)); + float _exp_286 = __nv_expf(_neg_287); + float2 rp_e_0 = make_float2(_exp_283, _exp_286); float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_285 = approx_rcp(rp_den_0.x); - float _rcp_286 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_287 = make_float2(_rcp_285, _rcp_286); + float _rcp_289 = approx_rcp(rp_den_0.x); + float _rcp_290 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_291 = make_float2(_rcp_289, _rcp_290); float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_287)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_291)); float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_288 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_292 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_288)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_292)); u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_289 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_293 = make_float2(gacc[4], gacc[4]); float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_289), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_292; - asm("abs.f32 %0, %1;" : "=f"(_abs_292) : "f"(rp_z_1.x)); - float _neg_291; - asm("neg.f32 %0, %1;" : "=f"(_neg_291) : "f"(_abs_292)); - float _exp_290 = __nv_expf(_neg_291); - float _abs_295; - asm("abs.f32 %0, %1;" : "=f"(_abs_295) : "f"(rp_z_1.y)); - float _neg_294; - asm("neg.f32 %0, %1;" : "=f"(_neg_294) : "f"(_abs_295)); - float _exp_293 = __nv_expf(_neg_294); - float2 rp_e_1 = make_float2(_exp_290, _exp_293); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_293), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_296; + asm("abs.f32 %0, %1;" : "=f"(_abs_296) : "f"(rp_z_1.x)); + float _neg_295; + asm("neg.f32 %0, %1;" : "=f"(_neg_295) : "f"(_abs_296)); + float _exp_294 = __nv_expf(_neg_295); + float _abs_299; + asm("abs.f32 %0, %1;" : "=f"(_abs_299) : "f"(rp_z_1.y)); + float _neg_298; + asm("neg.f32 %0, %1;" : "=f"(_neg_298) : "f"(_abs_299)); + float _exp_297 = __nv_expf(_neg_298); + float2 rp_e_1 = make_float2(_exp_294, _exp_297); float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_296 = approx_rcp(rp_den_1.x); - float _rcp_297 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_298 = make_float2(_rcp_296, _rcp_297); + float _rcp_300 = approx_rcp(rp_den_1.x); + float _rcp_301 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_302 = make_float2(_rcp_300, _rcp_301); float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_298)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_302)); float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_299 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_303 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_299)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_303)); u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_300 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_304 = make_float2(gacc[4], gacc[4]); float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_303; - asm("abs.f32 %0, %1;" : "=f"(_abs_303) : "f"(rp_z_2.x)); - float _neg_302; - asm("neg.f32 %0, %1;" : "=f"(_neg_302) : "f"(_abs_303)); - float _exp_301 = __nv_expf(_neg_302); - float _abs_306; - asm("abs.f32 %0, %1;" : "=f"(_abs_306) : "f"(rp_z_2.y)); - float _neg_305; - asm("neg.f32 %0, %1;" : "=f"(_neg_305) : "f"(_abs_306)); - float _exp_304 = __nv_expf(_neg_305); - float2 rp_e_2 = make_float2(_exp_301, _exp_304); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_304), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_307; + asm("abs.f32 %0, %1;" : "=f"(_abs_307) : "f"(rp_z_2.x)); + float _neg_306; + asm("neg.f32 %0, %1;" : "=f"(_neg_306) : "f"(_abs_307)); + float _exp_305 = __nv_expf(_neg_306); + float _abs_310; + asm("abs.f32 %0, %1;" : "=f"(_abs_310) : "f"(rp_z_2.y)); + float _neg_309; + asm("neg.f32 %0, %1;" : "=f"(_neg_309) : "f"(_abs_310)); + float _exp_308 = __nv_expf(_neg_309); + float2 rp_e_2 = make_float2(_exp_305, _exp_308); float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_307 = approx_rcp(rp_den_2.x); - float _rcp_308 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_309 = make_float2(_rcp_307, _rcp_308); + float _rcp_311 = approx_rcp(rp_den_2.x); + float _rcp_312 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_313 = make_float2(_rcp_311, _rcp_312); float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_313)); float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_310 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_314 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_310)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_314)); u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -3976,54 +3551,54 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_311; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_311) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_311; + uint32_t _prmt_315; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_315; int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_312 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = pq_0[0]; + auto _gst_316 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = pq_0[0]; } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_313; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_313) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_313; + uint32_t _prmt_317; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_317; int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_314 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = pq_1[0]; + auto _gst_318 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = pq_1[0]; } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_315; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_315; + uint32_t _prmt_319; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_319) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_319; int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_316 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = pq_2[0]; + auto _gst_320 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = pq_2[0]; } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_317; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_317; + uint32_t _prmt_321; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_321) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_321; int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_318 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = pq_3[0]; + auto _gst_322 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast(_gst_322) + (0)) = pq_3[0]; } } int t_a_0 = 48 + 8 * th; @@ -4088,47 +3663,47 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_319; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_319) : "f"(s_a), "f"(nm_t)); + float _sub_323; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_323) : "f"(s_a), "f"(nm_t)); float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_319), "f"(pa)); - float _sub_320; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_320) : "f"(pa), "f"(nm_t)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_323), "f"(pa)); + float _sub_324; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_324) : "f"(pa), "f"(nm_t)); float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_320), "f"(s_a)); - float _abs_322; - asm("abs.f32 %0, %1;" : "=f"(_abs_322) : "f"(s_a)); - float _abs_323; - asm("abs.f32 %0, %1;" : "=f"(_abs_323) : "f"(pa)); - uint32_t _setp_321_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_321_bits) : "f"(_abs_322), "f"(_abs_323)); - bool _setp_321 = _setp_321_bits != 0; - float nm_err = ((_setp_321) ? nm_es : nm_ex); - float _add_324; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(c_a), "f"(nm_err)); - c_a = _add_324; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_324), "f"(s_a)); + float _abs_326; + asm("abs.f32 %0, %1;" : "=f"(_abs_326) : "f"(s_a)); + float _abs_327; + asm("abs.f32 %0, %1;" : "=f"(_abs_327) : "f"(pa)); + uint32_t _setp_325_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_325_bits) : "f"(_abs_326), "f"(_abs_327)); + bool _setp_325 = _setp_325_bits != 0; + float nm_err = ((_setp_325) ? nm_es : nm_ex); + float _add_328; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(c_a), "f"(nm_err)); + c_a = _add_328; s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_325; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_325) : "f"(s_b), "f"(nm_t_0)); + float _sub_329; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_329) : "f"(s_b), "f"(nm_t_0)); float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_325), "f"(pb)); - float _sub_326; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_326) : "f"(pb), "f"(nm_t_0)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_329), "f"(pb)); + float _sub_330; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_330) : "f"(pb), "f"(nm_t_0)); float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_326), "f"(s_b)); - float _abs_328; - asm("abs.f32 %0, %1;" : "=f"(_abs_328) : "f"(s_b)); - float _abs_329; - asm("abs.f32 %0, %1;" : "=f"(_abs_329) : "f"(pb)); - uint32_t _setp_327_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_327_bits) : "f"(_abs_328), "f"(_abs_329)); - bool _setp_327 = _setp_327_bits != 0; - float nm_err_0 = ((_setp_327) ? nm_es_0 : nm_ex_0); - float _add_330; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_330; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_330), "f"(s_b)); + float _abs_332; + asm("abs.f32 %0, %1;" : "=f"(_abs_332) : "f"(s_b)); + float _abs_333; + asm("abs.f32 %0, %1;" : "=f"(_abs_333) : "f"(pb)); + uint32_t _setp_331_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_331_bits) : "f"(_abs_332), "f"(_abs_333)); + bool _setp_331 = _setp_331_bits != 0; + float nm_err_0 = ((_setp_331) ? nm_es_0 : nm_ex_0); + float _add_334; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_334; s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4152,132 +3727,132 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_331 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_335 = make_float2(gacc[4], gacc[4]); float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_331), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_334; - asm("abs.f32 %0, %1;" : "=f"(_abs_334) : "f"(rp_z_3.x)); - float _neg_333; - asm("neg.f32 %0, %1;" : "=f"(_neg_333) : "f"(_abs_334)); - float _exp_332 = __nv_expf(_neg_333); - float _abs_337; - asm("abs.f32 %0, %1;" : "=f"(_abs_337) : "f"(rp_z_3.y)); - float _neg_336; - asm("neg.f32 %0, %1;" : "=f"(_neg_336) : "f"(_abs_337)); - float _exp_335 = __nv_expf(_neg_336); - float2 rp_e_3 = make_float2(_exp_332, _exp_335); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_335), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_338; + asm("abs.f32 %0, %1;" : "=f"(_abs_338) : "f"(rp_z_3.x)); + float _neg_337; + asm("neg.f32 %0, %1;" : "=f"(_neg_337) : "f"(_abs_338)); + float _exp_336 = __nv_expf(_neg_337); + float _abs_341; + asm("abs.f32 %0, %1;" : "=f"(_abs_341) : "f"(rp_z_3.y)); + float _neg_340; + asm("neg.f32 %0, %1;" : "=f"(_neg_340) : "f"(_abs_341)); + float _exp_339 = __nv_expf(_neg_340); + float2 rp_e_3 = make_float2(_exp_336, _exp_339); float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_338 = approx_rcp(rp_den_3.x); - float _rcp_339 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_340 = make_float2(_rcp_338, _rcp_339); + float _rcp_342 = approx_rcp(rp_den_3.x); + float _rcp_343 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_344 = make_float2(_rcp_342, _rcp_343); float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_340)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_344)); float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_341 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_345 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_345)); u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_342 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_346 = make_float2(gacc[4], gacc[4]); float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_342), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_345; - asm("abs.f32 %0, %1;" : "=f"(_abs_345) : "f"(rp_z_4.x)); - float _neg_344; - asm("neg.f32 %0, %1;" : "=f"(_neg_344) : "f"(_abs_345)); - float _exp_343 = __nv_expf(_neg_344); - float _abs_348; - asm("abs.f32 %0, %1;" : "=f"(_abs_348) : "f"(rp_z_4.y)); - float _neg_347; - asm("neg.f32 %0, %1;" : "=f"(_neg_347) : "f"(_abs_348)); - float _exp_346 = __nv_expf(_neg_347); - float2 rp_e_4 = make_float2(_exp_343, _exp_346); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_346), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_349; + asm("abs.f32 %0, %1;" : "=f"(_abs_349) : "f"(rp_z_4.x)); + float _neg_348; + asm("neg.f32 %0, %1;" : "=f"(_neg_348) : "f"(_abs_349)); + float _exp_347 = __nv_expf(_neg_348); + float _abs_352; + asm("abs.f32 %0, %1;" : "=f"(_abs_352) : "f"(rp_z_4.y)); + float _neg_351; + asm("neg.f32 %0, %1;" : "=f"(_neg_351) : "f"(_abs_352)); + float _exp_350 = __nv_expf(_neg_351); + float2 rp_e_4 = make_float2(_exp_347, _exp_350); float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_349 = approx_rcp(rp_den_4.x); - float _rcp_350 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_351 = make_float2(_rcp_349, _rcp_350); + float _rcp_353 = approx_rcp(rp_den_4.x); + float _rcp_354 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_355 = make_float2(_rcp_353, _rcp_354); float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_351)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_355)); float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_352 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_356 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_352)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_356)); u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_353 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_357 = make_float2(gacc[4], gacc[4]); float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_356; - asm("abs.f32 %0, %1;" : "=f"(_abs_356) : "f"(rp_z_5.x)); - float _neg_355; - asm("neg.f32 %0, %1;" : "=f"(_neg_355) : "f"(_abs_356)); - float _exp_354 = __nv_expf(_neg_355); - float _abs_359; - asm("abs.f32 %0, %1;" : "=f"(_abs_359) : "f"(rp_z_5.y)); - float _neg_358; - asm("neg.f32 %0, %1;" : "=f"(_neg_358) : "f"(_abs_359)); - float _exp_357 = __nv_expf(_neg_358); - float2 rp_e_5 = make_float2(_exp_354, _exp_357); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_357), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_360; + asm("abs.f32 %0, %1;" : "=f"(_abs_360) : "f"(rp_z_5.x)); + float _neg_359; + asm("neg.f32 %0, %1;" : "=f"(_neg_359) : "f"(_abs_360)); + float _exp_358 = __nv_expf(_neg_359); + float _abs_363; + asm("abs.f32 %0, %1;" : "=f"(_abs_363) : "f"(rp_z_5.y)); + float _neg_362; + asm("neg.f32 %0, %1;" : "=f"(_neg_362) : "f"(_abs_363)); + float _exp_361 = __nv_expf(_neg_362); + float2 rp_e_5 = make_float2(_exp_358, _exp_361); float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_360 = approx_rcp(rp_den_5.x); - float _rcp_361 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_362 = make_float2(_rcp_360, _rcp_361); + float _rcp_364 = approx_rcp(rp_den_5.x); + float _rcp_365 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_366 = make_float2(_rcp_364, _rcp_365); float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_362)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_366)); float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_363 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_367 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_363)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_367)); u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_364 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_368 = make_float2(gacc[4], gacc[4]); float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_364), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_367; - asm("abs.f32 %0, %1;" : "=f"(_abs_367) : "f"(rp_z_6.x)); - float _neg_366; - asm("neg.f32 %0, %1;" : "=f"(_neg_366) : "f"(_abs_367)); - float _exp_365 = __nv_expf(_neg_366); - float _abs_370; - asm("abs.f32 %0, %1;" : "=f"(_abs_370) : "f"(rp_z_6.y)); - float _neg_369; - asm("neg.f32 %0, %1;" : "=f"(_neg_369) : "f"(_abs_370)); - float _exp_368 = __nv_expf(_neg_369); - float2 rp_e_6 = make_float2(_exp_365, _exp_368); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_368), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_371; + asm("abs.f32 %0, %1;" : "=f"(_abs_371) : "f"(rp_z_6.x)); + float _neg_370; + asm("neg.f32 %0, %1;" : "=f"(_neg_370) : "f"(_abs_371)); + float _exp_369 = __nv_expf(_neg_370); + float _abs_374; + asm("abs.f32 %0, %1;" : "=f"(_abs_374) : "f"(rp_z_6.y)); + float _neg_373; + asm("neg.f32 %0, %1;" : "=f"(_neg_373) : "f"(_abs_374)); + float _exp_372 = __nv_expf(_neg_373); + float2 rp_e_6 = make_float2(_exp_369, _exp_372); float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_371 = approx_rcp(rp_den_6.x); - float _rcp_372 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_373 = make_float2(_rcp_371, _rcp_372); + float _rcp_375 = approx_rcp(rp_den_6.x); + float _rcp_376 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_377 = make_float2(_rcp_375, _rcp_376); float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_373)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_377)); float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_374 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_378 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_374)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_378)); u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4292,54 +3867,54 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_375; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_375) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_375; + uint32_t _prmt_379; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_379; int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_376 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_376)); - *(reinterpret_cast(_gst_376) + (0)) = pq_4[0]; + auto _gst_380 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = pq_4[0]; } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_377; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_377) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_377; + uint32_t _prmt_381; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_381; int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_378 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_378)); - *(reinterpret_cast(_gst_378) + (0)) = pq_5[0]; + auto _gst_382 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = pq_5[0]; } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_379; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_379; + uint32_t _prmt_383; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_383) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_383; int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_380 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_380)); - *(reinterpret_cast(_gst_380) + (0)) = pq_6[0]; + auto _gst_384 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = pq_6[0]; } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_381; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_381; + uint32_t _prmt_385; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_385) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_385; int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_382 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_382)); - *(reinterpret_cast(_gst_382) + (0)) = pq_7[0]; + auto _gst_386 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_386)); + *(reinterpret_cast(_gst_386) + (0)) = pq_7[0]; } } int t_a_8 = 32 + 8 * th; @@ -4404,47 +3979,47 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_383; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_383) : "f"(s_a_0), "f"(nm_t_1)); + float _sub_387; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_387) : "f"(s_a_0), "f"(nm_t_1)); float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_383), "f"(pa_0)); - float _sub_384; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_384) : "f"(pa_0), "f"(nm_t_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_387), "f"(pa_0)); + float _sub_388; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_388) : "f"(pa_0), "f"(nm_t_1)); float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_384), "f"(s_a_0)); - float _abs_386; - asm("abs.f32 %0, %1;" : "=f"(_abs_386) : "f"(s_a_0)); - float _abs_387; - asm("abs.f32 %0, %1;" : "=f"(_abs_387) : "f"(pa_0)); - uint32_t _setp_385_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_385_bits) : "f"(_abs_386), "f"(_abs_387)); - bool _setp_385 = _setp_385_bits != 0; - float nm_err_1 = ((_setp_385) ? nm_es_1 : nm_ex_1); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_388), "f"(s_a_0)); + float _abs_390; + asm("abs.f32 %0, %1;" : "=f"(_abs_390) : "f"(s_a_0)); + float _abs_391; + asm("abs.f32 %0, %1;" : "=f"(_abs_391) : "f"(pa_0)); + uint32_t _setp_389_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_389_bits) : "f"(_abs_390), "f"(_abs_391)); + bool _setp_389 = _setp_389_bits != 0; + float nm_err_1 = ((_setp_389) ? nm_es_1 : nm_ex_1); + float _add_392; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_392; s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_389; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_389) : "f"(s_b_0), "f"(nm_t_2)); + float _sub_393; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_393) : "f"(s_b_0), "f"(nm_t_2)); float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_389), "f"(pb_0)); - float _sub_390; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_390) : "f"(pb_0), "f"(nm_t_2)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_393), "f"(pb_0)); + float _sub_394; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_394) : "f"(pb_0), "f"(nm_t_2)); float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_390), "f"(s_b_0)); - float _abs_392; - asm("abs.f32 %0, %1;" : "=f"(_abs_392) : "f"(s_b_0)); - float _abs_393; - asm("abs.f32 %0, %1;" : "=f"(_abs_393) : "f"(pb_0)); - uint32_t _setp_391_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_391_bits) : "f"(_abs_392), "f"(_abs_393)); - bool _setp_391 = _setp_391_bits != 0; - float nm_err_2 = ((_setp_391) ? nm_es_2 : nm_ex_2); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_394; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_394), "f"(s_b_0)); + float _abs_396; + asm("abs.f32 %0, %1;" : "=f"(_abs_396) : "f"(s_b_0)); + float _abs_397; + asm("abs.f32 %0, %1;" : "=f"(_abs_397) : "f"(pb_0)); + uint32_t _setp_395_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_395_bits) : "f"(_abs_396), "f"(_abs_397)); + bool _setp_395 = _setp_395_bits != 0; + float nm_err_2 = ((_setp_395) ? nm_es_2 : nm_ex_2); + float _add_398; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_398; s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4452,8 +4027,8 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_395 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_395), "f"(off_acc[0]) : "memory"); + uint32_t _addr_399 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_399), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4479,9 +4054,9 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1 = r_297[0]; + float r_255[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1 = r_255[0]; float gc_2 = gn_1 - 21.640425f; int eb_1 = 81920 + th * -32768; int ew_1 = eb_1 >> 2; @@ -4490,368 +4065,408 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int gb8_1 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1 * 128); int kb8_1 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1 * 128); float dbp_1[8]; - float qo_1[8]; - tmem_ld_x8(&qo_1[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); - float ko_1[8]; - tmem_ld_x8(&ko_1[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); - float qdg_1[8]; - tmem_ld_x8(&qdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); - float kdg_1[8]; - tmem_ld_x8(&kdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); + float qo_2[8]; + tmem_ld_x8(&qo_2[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); + float ko_2[8]; + tmem_ld_x8(&ko_2[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); + float qdg_2[8]; + tmem_ld_x8(&qdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); + float kdg_2[8]; + tmem_ld_x8(&kdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); float dkt_2[8]; tmem_ld_x8(&dkt_2[0], taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th)); - float btv_1[8]; + float btv_2[8]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1[0])), "=r"(*reinterpret_cast(&btv_1[(0) + 1])), "=r"(*reinterpret_cast(&btv_1[(0) + 2])), "=r"(*reinterpret_cast(&btv_1[(0) + 3])) + : "=r"(*reinterpret_cast(&btv_2[0])), "=r"(*reinterpret_cast(&btv_2[(0) + 1])), "=r"(*reinterpret_cast(&btv_2[(0) + 2])), "=r"(*reinterpret_cast(&btv_2[(0) + 3])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1[4])), "=r"(*reinterpret_cast(&btv_1[(4) + 1])), "=r"(*reinterpret_cast(&btv_1[(4) + 2])), "=r"(*reinterpret_cast(&btv_1[(4) + 3])) + : "=r"(*reinterpret_cast(&btv_2[4])), "=r"(*reinterpret_cast(&btv_2[(4) + 1])), "=r"(*reinterpret_cast(&btv_2[(4) + 2])), "=r"(*reinterpret_cast(&btv_2[(4) + 3])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th) + 16))); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1[1]; s_run_1[0] = 0.0f; - int t_16 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1 + 2944 + d_lane]; - float bt_t_16 = btv_1[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1); - float e_dg_16 = approx_exp2(g_t_16 - gc_2); - float dq2_16 = __fmaf_rn(qo_1[7], e_off_16, qdg_1[7] * e_dg_16); - float dk2_16 = __fmaf_rn(ko_1[7], e_off_16, kdg_1[7] * e_dg_16); - dbp_1[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + dkt_2[7]; - float dg_o_16 = __fmaf_rn(q_t_1_16, dq2_16, (dk2b_16 - dkt_2[7]) * k_t_16) + dgf_16; - s_run_1[0] = s_run_1[0] + dg_o_16; + int t_0 = 16 + 8 * th + 7; + float r_256[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_0 = r_256[0]; + unsigned int r_257[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_0 = __uint_as_float(r_257[0] << 16); + unsigned int r_258[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_0 = __uint_as_float(r_258[0] << 16); + float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); + float bt_t_0 = btv_2[7]; + float e_off_0 = approx_exp2(g_t_0 - gn_1); + float e_dg_0 = approx_exp2(g_t_0 - gc_2); + float dq2_0 = __fmaf_rn(qo_2[7], e_off_0, qdg_2[7] * e_dg_0); + float dk2_0 = __fmaf_rn(ko_2[7], e_off_0, kdg_2[7] * e_dg_0); + dbp_1[7] = dk2_0 * k_t_0; + float dk2b_0 = dk2_0 * bt_t_0; + float qe_2 = q_t_1_0 * e_dg_0; + float kp_13 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_13; + float kbe_2 = (float)kb_2 * e_dg_0; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg_2[7], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_13 * e_dg_0) * kdg_2[7]); + float dqf_0 = pool_f32[ew_1 + 896 + d_lane]; + float dkf_0 = pool_f32[ew_1 + 1920 + d_lane]; + float dgf_0 = pool_f32[ew_1 + 2944 + d_lane]; + float dq_o_0 = dq2_0 + dqf_0; + float dk_o_0 = dk2b_0 + dkf_0 + dkt_2[7]; + float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_2[7], k_t_0, cr_2)) + dgf_0; + s_run_1[0] = s_run_1[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_16 = t_16 * qrs; - if (t_16 < clen) { + int qoff_0 = t_0 * qrs; + if (t_0 < clen) { { - auto _gst_396 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_396)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_396) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_400 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_400)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_397 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_397)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_397) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_401 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_401)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_0; } } - int t_17 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1 + 2816 + d_lane]; - float bt_t_17 = btv_1[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1); - float e_dg_17 = approx_exp2(g_t_17 - gc_2); - float dq2_17 = __fmaf_rn(qo_1[6], e_off_17, qdg_1[6] * e_dg_17); - float dk2_17 = __fmaf_rn(ko_1[6], e_off_17, kdg_1[6] * e_dg_17); - dbp_1[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + dkt_2[6]; - float dg_o_17 = __fmaf_rn(q_t_1_17, dq2_17, (dk2b_17 - dkt_2[6]) * k_t_17) + dgf_17; - s_run_1[0] = s_run_1[0] + dg_o_17; + int t_1_1 = 16 + 8 * th + 6; + float r_259[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1_1 = r_259[0]; + unsigned int r_260[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1_1 = __uint_as_float(r_260[0] << 16); + unsigned int r_261[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1_2 = __uint_as_float(r_261[0] << 16); + float q_t_1_1_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_2); + float bt_t_1_1 = btv_2[6]; + float e_off_1_1 = approx_exp2(g_t_1_1 - gn_1); + float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_2); + float dq2_1_1 = __fmaf_rn(qo_2[6], e_off_1_1, qdg_2[6] * e_dg_1_1); + float dk2_1_1 = __fmaf_rn(ko_2[6], e_off_1_1, kdg_2[6] * e_dg_1_1); + dbp_1[6] = dk2_1_1 * k_t_1_1; + float dk2b_1_1 = dk2_1_1 * bt_t_1_1; + float qe_0 = q_t_1_1_1 * e_dg_1_1; + float kp_12_1 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12_1; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg_2[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12_1 * e_dg_1_1) * kdg_2[6]); + float dqf_1_1 = pool_f32[ew_1 + 768 + d_lane]; + float dkf_1_1 = pool_f32[ew_1 + 1792 + d_lane]; + float dgf_1_1 = pool_f32[ew_1 + 2816 + d_lane]; + float dq_o_1_1 = dq2_1_1 + dqf_1_1; + float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + dkt_2[6]; + float dg_o_1_1 = __fmaf_rn(q_t_1_1_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - dkt_2[6], k_t_1_1, cr_0)) + dgf_1_1; + s_run_1[0] = s_run_1[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_17 = t_17 * qrs; - if (t_17 < clen) { + int qoff_1_1 = t_1_1 * qrs; + if (t_1_1 < clen) { { - auto _gst_398 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_398)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_398) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_402 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_402)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_1_1; } { - auto _gst_399 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_399)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_399) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_403 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_403)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_1_1; } } - int t_18 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1 + 2688 + d_lane]; - float bt_t_18 = btv_1[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1); - float e_dg_18 = approx_exp2(g_t_18 - gc_2); - float dq2_18 = __fmaf_rn(qo_1[5], e_off_18, qdg_1[5] * e_dg_18); - float dk2_18 = __fmaf_rn(ko_1[5], e_off_18, kdg_1[5] * e_dg_18); - dbp_1[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + dkt_2[5]; - float dg_o_18 = __fmaf_rn(q_t_1_18, dq2_18, (dk2b_18 - dkt_2[5]) * k_t_18) + dgf_18; - s_run_1[0] = s_run_1[0] + dg_o_18; + int t_2 = 16 + 8 * th + 5; + float r_262[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_262[0]; + unsigned int r_263[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_263[0] << 16); + unsigned int r_264[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2_1 = __uint_as_float(r_264[0] << 16); + float q_t_1_2_1 = ((t_2 >= clen) ? 0.0f : q_t_2_1); + float bt_t_2 = btv_2[5]; + float e_off_2 = approx_exp2(g_t_2 - gn_1); + float e_dg_2 = approx_exp2(g_t_2 - gc_2); + float dq2_2 = __fmaf_rn(qo_2[5], e_off_2, qdg_2[5] * e_dg_2); + float dk2_2 = __fmaf_rn(ko_2[5], e_off_2, kdg_2[5] * e_dg_2); + dbp_1[5] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float qe_1_1 = q_t_1_2_1 * e_dg_2; + float kp_13_1 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1_1 = (__nv_bfloat16)kp_13_1; + float kbe_1_1 = (float)kb_1_1 * e_dg_2; + float cr_1_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1_1) & 4294959104u) - qe_1_1, qdg_2[5], (__uint_as_float(__as_u32(kbe_1_1) & 4294959104u) - kp_13_1 * e_dg_2) * kdg_2[5]); + float dqf_2 = pool_f32[ew_1 + 640 + d_lane]; + float dkf_2 = pool_f32[ew_1 + 1664 + d_lane]; + float dgf_2 = pool_f32[ew_1 + 2688 + d_lane]; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + dkt_2[5]; + float dg_o_2 = __fmaf_rn(q_t_1_2_1, dq2_2, __fmaf_rn(dk2b_2 - dkt_2[5], k_t_2, cr_1_1)) + dgf_2; + s_run_1[0] = s_run_1[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_18 = t_18 * qrs; - if (t_18 < clen) { + int qoff_2 = t_2 * qrs; + if (t_2 < clen) { { - auto _gst_400 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_400)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_404 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_404)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_401 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_401)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_405 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_405)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_2; } } - int t_19 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1 + 2560 + d_lane]; - float bt_t_19 = btv_1[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1); - float e_dg_19 = approx_exp2(g_t_19 - gc_2); - float dq2_19 = __fmaf_rn(qo_1[4], e_off_19, qdg_1[4] * e_dg_19); - float dk2_19 = __fmaf_rn(ko_1[4], e_off_19, kdg_1[4] * e_dg_19); - dbp_1[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + dkt_2[4]; - float dg_o_19 = __fmaf_rn(q_t_1_19, dq2_19, (dk2b_19 - dkt_2[4]) * k_t_19) + dgf_19; - s_run_1[0] = s_run_1[0] + dg_o_19; + int t_3 = 16 + 8 * th + 4; + float r_265[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_3 = r_265[0]; + unsigned int r_266[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_3 = __uint_as_float(r_266[0] << 16); + unsigned int r_267[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_3 = __uint_as_float(r_267[0] << 16); + float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); + float bt_t_3 = btv_2[4]; + float e_off_3 = approx_exp2(g_t_3 - gn_1); + float e_dg_3 = approx_exp2(g_t_3 - gc_2); + float dq2_3 = __fmaf_rn(qo_2[4], e_off_3, qdg_2[4] * e_dg_3); + float dk2_3 = __fmaf_rn(ko_2[4], e_off_3, kdg_2[4] * e_dg_3); + dbp_1[4] = dk2_3 * k_t_3; + float dk2b_3 = dk2_3 * bt_t_3; + float qe_2_1 = q_t_1_3 * e_dg_3; + float kp_14 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2_1 = (__nv_bfloat16)kp_14; + float kbe_2_1 = (float)kb_2_1 * e_dg_3; + float cr_2_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_2_1) & 4294959104u) - qe_2_1, qdg_2[4], (__uint_as_float(__as_u32(kbe_2_1) & 4294959104u) - kp_14 * e_dg_3) * kdg_2[4]); + float dqf_3 = pool_f32[ew_1 + 512 + d_lane]; + float dkf_3 = pool_f32[ew_1 + 1536 + d_lane]; + float dgf_3 = pool_f32[ew_1 + 2560 + d_lane]; + float dq_o_3 = dq2_3 + dqf_3; + float dk_o_3 = dk2b_3 + dkf_3 + dkt_2[4]; + float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_2[4], k_t_3, cr_2_1)) + dgf_3; + s_run_1[0] = s_run_1[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_19 = t_19 * qrs; - if (t_19 < clen) { + int qoff_3 = t_3 * qrs; + if (t_3 < clen) { { - auto _gst_402 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_402)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_406 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_406)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_403 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_403)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_407 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_407)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_3; } } - int t_20 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1 + 2432 + d_lane]; - float bt_t_20 = btv_1[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1); - float e_dg_20 = approx_exp2(g_t_20 - gc_2); - float dq2_20 = __fmaf_rn(qo_1[3], e_off_20, qdg_1[3] * e_dg_20); - float dk2_20 = __fmaf_rn(ko_1[3], e_off_20, kdg_1[3] * e_dg_20); - dbp_1[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + dkt_2[3]; - float dg_o_20 = __fmaf_rn(q_t_1_20, dq2_20, (dk2b_20 - dkt_2[3]) * k_t_20) + dgf_20; - s_run_1[0] = s_run_1[0] + dg_o_20; + int t_4 = 16 + 8 * th + 3; + float r_268[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_4 = r_268[0]; + unsigned int r_269[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_4 = __uint_as_float(r_269[0] << 16); + unsigned int r_270[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_4 = __uint_as_float(r_270[0] << 16); + float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); + float bt_t_4 = btv_2[3]; + float e_off_4 = approx_exp2(g_t_4 - gn_1); + float e_dg_4 = approx_exp2(g_t_4 - gc_2); + float dq2_4 = __fmaf_rn(qo_2[3], e_off_4, qdg_2[3] * e_dg_4); + float dk2_4 = __fmaf_rn(ko_2[3], e_off_4, kdg_2[3] * e_dg_4); + dbp_1[3] = dk2_4 * k_t_4; + float dk2b_4 = dk2_4 * bt_t_4; + float qe_3 = q_t_1_4 * e_dg_4; + float kp_15 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg_2[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_4) * kdg_2[3]); + float dqf_4 = pool_f32[ew_1 + 384 + d_lane]; + float dkf_4 = pool_f32[ew_1 + 1408 + d_lane]; + float dgf_4 = pool_f32[ew_1 + 2432 + d_lane]; + float dq_o_4 = dq2_4 + dqf_4; + float dk_o_4 = dk2b_4 + dkf_4 + dkt_2[3]; + float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_2[3], k_t_4, cr_3)) + dgf_4; + s_run_1[0] = s_run_1[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_20 = t_20 * qrs; - if (t_20 < clen) { + int qoff_4 = t_4 * qrs; + if (t_4 < clen) { { - auto _gst_404 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_404)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_408 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_408)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_405 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_405)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_409 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_409)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_4; } } - int t_21 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1 + 2304 + d_lane]; - float bt_t_21 = btv_1[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1); - float e_dg_21 = approx_exp2(g_t_21 - gc_2); - float dq2_21 = __fmaf_rn(qo_1[2], e_off_21, qdg_1[2] * e_dg_21); - float dk2_21 = __fmaf_rn(ko_1[2], e_off_21, kdg_1[2] * e_dg_21); - dbp_1[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + dkt_2[2]; - float dg_o_21 = __fmaf_rn(q_t_1_21, dq2_21, (dk2b_21 - dkt_2[2]) * k_t_21) + dgf_21; - s_run_1[0] = s_run_1[0] + dg_o_21; + int t_5 = 16 + 8 * th + 2; + float r_271[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_5 = r_271[0]; + unsigned int r_272[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_5 = __uint_as_float(r_272[0] << 16); + unsigned int r_273[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_5 = __uint_as_float(r_273[0] << 16); + float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); + float bt_t_5 = btv_2[2]; + float e_off_5 = approx_exp2(g_t_5 - gn_1); + float e_dg_5 = approx_exp2(g_t_5 - gc_2); + float dq2_5 = __fmaf_rn(qo_2[2], e_off_5, qdg_2[2] * e_dg_5); + float dk2_5 = __fmaf_rn(ko_2[2], e_off_5, kdg_2[2] * e_dg_5); + dbp_1[2] = dk2_5 * k_t_5; + float dk2b_5 = dk2_5 * bt_t_5; + float qe_4 = q_t_1_5 * e_dg_5; + float kp_16 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg_2[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_5) * kdg_2[2]); + float dqf_5 = pool_f32[ew_1 + 256 + d_lane]; + float dkf_5 = pool_f32[ew_1 + 1280 + d_lane]; + float dgf_5 = pool_f32[ew_1 + 2304 + d_lane]; + float dq_o_5 = dq2_5 + dqf_5; + float dk_o_5 = dk2b_5 + dkf_5 + dkt_2[2]; + float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_2[2], k_t_5, cr_4)) + dgf_5; + s_run_1[0] = s_run_1[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_21 = t_21 * qrs; - if (t_21 < clen) { + int qoff_5 = t_5 * qrs; + if (t_5 < clen) { { - auto _gst_406 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_406)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_410 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_410)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_407 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_407)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_411 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_411)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_5; } } - int t_22 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1 + 2176 + d_lane]; - float bt_t_22 = btv_1[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1); - float e_dg_22 = approx_exp2(g_t_22 - gc_2); - float dq2_22 = __fmaf_rn(qo_1[1], e_off_22, qdg_1[1] * e_dg_22); - float dk2_22 = __fmaf_rn(ko_1[1], e_off_22, kdg_1[1] * e_dg_22); - dbp_1[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + dkt_2[1]; - float dg_o_22 = __fmaf_rn(q_t_1_22, dq2_22, (dk2b_22 - dkt_2[1]) * k_t_22) + dgf_22; - s_run_1[0] = s_run_1[0] + dg_o_22; + int t_6 = 16 + 8 * th + 1; + float r_274[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_6 = r_274[0]; + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_6 = __uint_as_float(r_275[0] << 16); + unsigned int r_276[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_6 = __uint_as_float(r_276[0] << 16); + float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); + float bt_t_6 = btv_2[1]; + float e_off_6 = approx_exp2(g_t_6 - gn_1); + float e_dg_6 = approx_exp2(g_t_6 - gc_2); + float dq2_6 = __fmaf_rn(qo_2[1], e_off_6, qdg_2[1] * e_dg_6); + float dk2_6 = __fmaf_rn(ko_2[1], e_off_6, kdg_2[1] * e_dg_6); + dbp_1[1] = dk2_6 * k_t_6; + float dk2b_6 = dk2_6 * bt_t_6; + float qe_5 = q_t_1_6 * e_dg_6; + float kp_17 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg_2[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_6) * kdg_2[1]); + float dqf_6 = pool_f32[ew_1 + 128 + d_lane]; + float dkf_6 = pool_f32[ew_1 + 1152 + d_lane]; + float dgf_6 = pool_f32[ew_1 + 2176 + d_lane]; + float dq_o_6 = dq2_6 + dqf_6; + float dk_o_6 = dk2b_6 + dkf_6 + dkt_2[1]; + float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_2[1], k_t_6, cr_5)) + dgf_6; + s_run_1[0] = s_run_1[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_22 = t_22 * qrs; - if (t_22 < clen) { + int qoff_6 = t_6 * qrs; + if (t_6 < clen) { { - auto _gst_408 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_408)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_412 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_412)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_412) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_409 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_409)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_413 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_413)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_413) + (0)) = (__nv_bfloat16)dk_o_6; } } - int t_23 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1 + d_lane]; - float dkf_23 = pool_f32[ew_1 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1 + 2048 + d_lane]; - float bt_t_23 = btv_1[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1); - float e_dg_23 = approx_exp2(g_t_23 - gc_2); - float dq2_23 = __fmaf_rn(qo_1[0], e_off_23, qdg_1[0] * e_dg_23); - float dk2_23 = __fmaf_rn(ko_1[0], e_off_23, kdg_1[0] * e_dg_23); - dbp_1[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + dkt_2[0]; - float dg_o_23 = __fmaf_rn(q_t_1_23, dq2_23, (dk2b_23 - dkt_2[0]) * k_t_23) + dgf_23; - s_run_1[0] = s_run_1[0] + dg_o_23; + int t_7 = 16 + 8 * th; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_7 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_7 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_7 = __uint_as_float(r_279[0] << 16); + float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); + float bt_t_7 = btv_2[0]; + float e_off_7 = approx_exp2(g_t_7 - gn_1); + float e_dg_7 = approx_exp2(g_t_7 - gc_2); + float dq2_7 = __fmaf_rn(qo_2[0], e_off_7, qdg_2[0] * e_dg_7); + float dk2_7 = __fmaf_rn(ko_2[0], e_off_7, kdg_2[0] * e_dg_7); + dbp_1[0] = dk2_7 * k_t_7; + float dk2b_7 = dk2_7 * bt_t_7; + float qe_6 = q_t_1_7 * e_dg_7; + float kp_18 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg_2[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_7) * kdg_2[0]); + float dqf_7 = pool_f32[ew_1 + d_lane]; + float dkf_7 = pool_f32[ew_1 + 1024 + d_lane]; + float dgf_7 = pool_f32[ew_1 + 2048 + d_lane]; + float dq_o_7 = dq2_7 + dqf_7; + float dk_o_7 = dk2b_7 + dkf_7 + dkt_2[0]; + float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_2[0], k_t_7, cr_6)) + dgf_7; + s_run_1[0] = s_run_1[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_23 = t_23 * qrs; - if (t_23 < clen) { + int qoff_7 = t_7 * qrs; + if (t_7 < clen) { { - auto _gst_410 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_410)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_414 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_414)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_414) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_411 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_411)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_415 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_415)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_415) + (0)) = (__nv_bfloat16)dk_o_7; } } float tot_1 = s_run_1[0]; @@ -4860,39 +4475,39 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_412 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_412; + float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_416; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_413; + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_417; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_414 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_414; + float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_418; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_415; + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_419; float v2_17[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_17[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_416; + float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_17[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_420; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_17[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_417; + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_17[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_421; float send_19 = ((b2_1 == 0) ? v2_17[1] : v2_17[0]); - float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_418; - float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_419; - float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_420; + float _shfl_xor_sync_422 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_422; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_423; + float _shfl_xor_sync_424 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_424; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_421 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_421), "f"(tot_1) : "memory"); + uint32_t _addr_425 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_425), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4924,132 +4539,132 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_422 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_426 = make_float2(gacc[4], gacc[4]); float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_422), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_425; - asm("abs.f32 %0, %1;" : "=f"(_abs_425) : "f"(rp_z_7.x)); - float _neg_424; - asm("neg.f32 %0, %1;" : "=f"(_neg_424) : "f"(_abs_425)); - float _exp_423 = __nv_expf(_neg_424); - float _abs_428; - asm("abs.f32 %0, %1;" : "=f"(_abs_428) : "f"(rp_z_7.y)); - float _neg_427; - asm("neg.f32 %0, %1;" : "=f"(_neg_427) : "f"(_abs_428)); - float _exp_426 = __nv_expf(_neg_427); - float2 rp_e_7 = make_float2(_exp_423, _exp_426); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_426), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_429; + asm("abs.f32 %0, %1;" : "=f"(_abs_429) : "f"(rp_z_7.x)); + float _neg_428; + asm("neg.f32 %0, %1;" : "=f"(_neg_428) : "f"(_abs_429)); + float _exp_427 = __nv_expf(_neg_428); + float _abs_432; + asm("abs.f32 %0, %1;" : "=f"(_abs_432) : "f"(rp_z_7.y)); + float _neg_431; + asm("neg.f32 %0, %1;" : "=f"(_neg_431) : "f"(_abs_432)); + float _exp_430 = __nv_expf(_neg_431); + float2 rp_e_7 = make_float2(_exp_427, _exp_430); float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_429 = approx_rcp(rp_den_7.x); - float _rcp_430 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_431 = make_float2(_rcp_429, _rcp_430); + float _rcp_433 = approx_rcp(rp_den_7.x); + float _rcp_434 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_435 = make_float2(_rcp_433, _rcp_434); float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_431)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_435)); float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_432 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_436 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_432)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_436)); u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_433 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_437 = make_float2(gacc[4], gacc[4]); float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_433), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_436; - asm("abs.f32 %0, %1;" : "=f"(_abs_436) : "f"(rp_z_8.x)); - float _neg_435; - asm("neg.f32 %0, %1;" : "=f"(_neg_435) : "f"(_abs_436)); - float _exp_434 = __nv_expf(_neg_435); - float _abs_439; - asm("abs.f32 %0, %1;" : "=f"(_abs_439) : "f"(rp_z_8.y)); - float _neg_438; - asm("neg.f32 %0, %1;" : "=f"(_neg_438) : "f"(_abs_439)); - float _exp_437 = __nv_expf(_neg_438); - float2 rp_e_8 = make_float2(_exp_434, _exp_437); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_437), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_440; + asm("abs.f32 %0, %1;" : "=f"(_abs_440) : "f"(rp_z_8.x)); + float _neg_439; + asm("neg.f32 %0, %1;" : "=f"(_neg_439) : "f"(_abs_440)); + float _exp_438 = __nv_expf(_neg_439); + float _abs_443; + asm("abs.f32 %0, %1;" : "=f"(_abs_443) : "f"(rp_z_8.y)); + float _neg_442; + asm("neg.f32 %0, %1;" : "=f"(_neg_442) : "f"(_abs_443)); + float _exp_441 = __nv_expf(_neg_442); + float2 rp_e_8 = make_float2(_exp_438, _exp_441); float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_440 = approx_rcp(rp_den_8.x); - float _rcp_441 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_442 = make_float2(_rcp_440, _rcp_441); + float _rcp_444 = approx_rcp(rp_den_8.x); + float _rcp_445 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_446 = make_float2(_rcp_444, _rcp_445); float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_442)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_446)); float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_443 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_447 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_443)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_447)); u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_444 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_448 = make_float2(gacc[4], gacc[4]); float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_444), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_447; - asm("abs.f32 %0, %1;" : "=f"(_abs_447) : "f"(rp_z_9.x)); - float _neg_446; - asm("neg.f32 %0, %1;" : "=f"(_neg_446) : "f"(_abs_447)); - float _exp_445 = __nv_expf(_neg_446); - float _abs_450; - asm("abs.f32 %0, %1;" : "=f"(_abs_450) : "f"(rp_z_9.y)); - float _neg_449; - asm("neg.f32 %0, %1;" : "=f"(_neg_449) : "f"(_abs_450)); - float _exp_448 = __nv_expf(_neg_449); - float2 rp_e_9 = make_float2(_exp_445, _exp_448); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_448), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_451; + asm("abs.f32 %0, %1;" : "=f"(_abs_451) : "f"(rp_z_9.x)); + float _neg_450; + asm("neg.f32 %0, %1;" : "=f"(_neg_450) : "f"(_abs_451)); + float _exp_449 = __nv_expf(_neg_450); + float _abs_454; + asm("abs.f32 %0, %1;" : "=f"(_abs_454) : "f"(rp_z_9.y)); + float _neg_453; + asm("neg.f32 %0, %1;" : "=f"(_neg_453) : "f"(_abs_454)); + float _exp_452 = __nv_expf(_neg_453); + float2 rp_e_9 = make_float2(_exp_449, _exp_452); float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_451 = approx_rcp(rp_den_9.x); - float _rcp_452 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_453 = make_float2(_rcp_451, _rcp_452); + float _rcp_455 = approx_rcp(rp_den_9.x); + float _rcp_456 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_457 = make_float2(_rcp_455, _rcp_456); float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_453)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_457)); float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_454 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_458 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_454)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_455 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_459 = make_float2(gacc[4], gacc[4]); float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_455), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_458; - asm("abs.f32 %0, %1;" : "=f"(_abs_458) : "f"(rp_z_10.x)); - float _neg_457; - asm("neg.f32 %0, %1;" : "=f"(_neg_457) : "f"(_abs_458)); - float _exp_456 = __nv_expf(_neg_457); - float _abs_461; - asm("abs.f32 %0, %1;" : "=f"(_abs_461) : "f"(rp_z_10.y)); - float _neg_460; - asm("neg.f32 %0, %1;" : "=f"(_neg_460) : "f"(_abs_461)); - float _exp_459 = __nv_expf(_neg_460); - float2 rp_e_10 = make_float2(_exp_456, _exp_459); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_459), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_462; + asm("abs.f32 %0, %1;" : "=f"(_abs_462) : "f"(rp_z_10.x)); + float _neg_461; + asm("neg.f32 %0, %1;" : "=f"(_neg_461) : "f"(_abs_462)); + float _exp_460 = __nv_expf(_neg_461); + float _abs_465; + asm("abs.f32 %0, %1;" : "=f"(_abs_465) : "f"(rp_z_10.y)); + float _neg_464; + asm("neg.f32 %0, %1;" : "=f"(_neg_464) : "f"(_abs_465)); + float _exp_463 = __nv_expf(_neg_464); + float2 rp_e_10 = make_float2(_exp_460, _exp_463); float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_462 = approx_rcp(rp_den_10.x); - float _rcp_463 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_464 = make_float2(_rcp_462, _rcp_463); + float _rcp_466 = approx_rcp(rp_den_10.x); + float _rcp_467 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_468 = make_float2(_rcp_466, _rcp_467); float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_464)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_468)); float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_465 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_469 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_465)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -5064,54 +4679,54 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_466; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_466) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_466; + uint32_t _prmt_470; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_470; int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_467 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_467)); - *(reinterpret_cast(_gst_467) + (0)) = pq_8[0]; + auto _gst_471 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_471)); + *(reinterpret_cast(_gst_471) + (0)) = pq_8[0]; } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_468; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_468) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_468; + uint32_t _prmt_472; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_472; int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_469 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_469)); - *(reinterpret_cast(_gst_469) + (0)) = pq_9[0]; + auto _gst_473 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_473)); + *(reinterpret_cast(_gst_473) + (0)) = pq_9[0]; } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_470; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_470; + uint32_t _prmt_474; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_474) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_474; int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_471 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_471)); - *(reinterpret_cast(_gst_471) + (0)) = pq_10[0]; + auto _gst_475 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_475)); + *(reinterpret_cast(_gst_475) + (0)) = pq_10[0]; } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_472; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_472; + uint32_t _prmt_476; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_476) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_476; int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_473 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_473)); - *(reinterpret_cast(_gst_473) + (0)) = pq_11[0]; + auto _gst_477 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_477)); + *(reinterpret_cast(_gst_477) + (0)) = pq_11[0]; } } int t_a_16 = 16 + 8 * th; @@ -5176,47 +4791,47 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_474; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_474) : "f"(s_a_1), "f"(nm_t_3)); + float _sub_478; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_478) : "f"(s_a_1), "f"(nm_t_3)); float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_474), "f"(pa_1)); - float _sub_475; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_475) : "f"(pa_1), "f"(nm_t_3)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_478), "f"(pa_1)); + float _sub_479; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_479) : "f"(pa_1), "f"(nm_t_3)); float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_475), "f"(s_a_1)); - float _abs_477; - asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(s_a_1)); - float _abs_478; - asm("abs.f32 %0, %1;" : "=f"(_abs_478) : "f"(pa_1)); - uint32_t _setp_476_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_476_bits) : "f"(_abs_477), "f"(_abs_478)); - bool _setp_476 = _setp_476_bits != 0; - float nm_err_3 = ((_setp_476) ? nm_es_3 : nm_ex_3); - float _add_479; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_479) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_479; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_479), "f"(s_a_1)); + float _abs_481; + asm("abs.f32 %0, %1;" : "=f"(_abs_481) : "f"(s_a_1)); + float _abs_482; + asm("abs.f32 %0, %1;" : "=f"(_abs_482) : "f"(pa_1)); + uint32_t _setp_480_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_480_bits) : "f"(_abs_481), "f"(_abs_482)); + bool _setp_480 = _setp_480_bits != 0; + float nm_err_3 = ((_setp_480) ? nm_es_3 : nm_ex_3); + float _add_483; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_483) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_483; s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_480; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_480) : "f"(s_b_1), "f"(nm_t_4)); + float _sub_484; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_484) : "f"(s_b_1), "f"(nm_t_4)); float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_480), "f"(pb_1)); - float _sub_481; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_481) : "f"(pb_1), "f"(nm_t_4)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_484), "f"(pb_1)); + float _sub_485; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_485) : "f"(pb_1), "f"(nm_t_4)); float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_481), "f"(s_b_1)); - float _abs_483; - asm("abs.f32 %0, %1;" : "=f"(_abs_483) : "f"(s_b_1)); - float _abs_484; - asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(pb_1)); - uint32_t _setp_482_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_482_bits) : "f"(_abs_483), "f"(_abs_484)); - bool _setp_482 = _setp_482_bits != 0; - float nm_err_4 = ((_setp_482) ? nm_es_4 : nm_ex_4); - float _add_485; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_485) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_485; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_485), "f"(s_b_1)); + float _abs_487; + asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(s_b_1)); + float _abs_488; + asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(pb_1)); + uint32_t _setp_486_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_486_bits) : "f"(_abs_487), "f"(_abs_488)); + bool _setp_486 = _setp_486_bits != 0; + float nm_err_4 = ((_setp_486) ? nm_es_4 : nm_ex_4); + float _add_489; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_489) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_489; s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5224,8 +4839,8 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_486 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_486), "f"(off_acc[0]) : "memory"); + uint32_t _addr_490 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_490), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5249,9 +4864,9 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_2 = r_322[0]; + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_2 = r_280[0]; float gc_3 = gn_2 - 21.640425f; int eb_2 = 98304 + th * -32768; int ew_2 = eb_2 >> 2; @@ -5260,364 +4875,404 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int gb8_2 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_2 * 128); int kb8_2 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_2 * 128); float dbp_2[8]; - float qdg_2[8]; - tmem_ld_x8(&qdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float kdg_2[8]; - tmem_ld_x8(&kdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); + float qdg_0[8]; + tmem_ld_x8(&qdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); + float kdg_0[8]; + tmem_ld_x8(&kdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); float dkt_3[8]; tmem_ld_x8(&dkt_3[0], taddr + (unsigned int)row_base + (unsigned int)(8 * th)); - float btv_2[8]; + float btv_0[8]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_2[0])), "=r"(*reinterpret_cast(&btv_2[(0) + 1])), "=r"(*reinterpret_cast(&btv_2[(0) + 2])), "=r"(*reinterpret_cast(&btv_2[(0) + 3])) + : "=r"(*reinterpret_cast(&btv_0[0])), "=r"(*reinterpret_cast(&btv_0[(0) + 1])), "=r"(*reinterpret_cast(&btv_0[(0) + 2])), "=r"(*reinterpret_cast(&btv_0[(0) + 3])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_2[4])), "=r"(*reinterpret_cast(&btv_2[(4) + 1])), "=r"(*reinterpret_cast(&btv_2[(4) + 2])), "=r"(*reinterpret_cast(&btv_2[(4) + 3])) + : "=r"(*reinterpret_cast(&btv_0[4])), "=r"(*reinterpret_cast(&btv_0[(4) + 1])), "=r"(*reinterpret_cast(&btv_0[(4) + 2])), "=r"(*reinterpret_cast(&btv_0[(4) + 3])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th) + 16))); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_2[1]; s_run_2[0] = 0.0f; - int t_24 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_24 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_24 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_24 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_24 >= clen) ? 0.0f : q_t_24); - float dqf_24 = pool_f32[ew_2 + 896 + d_lane]; - float dkf_24 = pool_f32[ew_2 + 1920 + d_lane]; - float dgf_24 = pool_f32[ew_2 + 2944 + d_lane]; - float bt_t_24 = btv_2[7]; - float e_off_24 = approx_exp2(g_t_24 - gn_2); - float e_dg_24 = approx_exp2(g_t_24 - gc_3); - float dq2_24 = qdg_2[7] * e_dg_24; - float dk2_24 = kdg_2[7] * e_dg_24; - dbp_2[7] = dk2_24 * k_t_24; - float dk2b_24 = dk2_24 * bt_t_24; - float dq_o_24 = dq2_24 + dqf_24; - float dk_o_24 = dk2b_24 + dkf_24 + dkt_3[7]; - float dg_o_24 = __fmaf_rn(q_t_1_24, dq2_24, (dk2b_24 - dkt_3[7]) * k_t_24) + dgf_24; - s_run_2[0] = s_run_2[0] + dg_o_24; + int t_8 = 8 * th + 7; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_8 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_8 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_8 = __uint_as_float(r_283[0] << 16); + float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); + float bt_t_8 = btv_0[7]; + float e_off_8 = approx_exp2(g_t_8 - gn_2); + float e_dg_8 = approx_exp2(g_t_8 - gc_3); + float dq2_8 = qdg_0[7] * e_dg_8; + float dk2_8 = kdg_0[7] * e_dg_8; + dbp_2[7] = dk2_8 * k_t_8; + float dk2b_8 = dk2_8 * bt_t_8; + float qe_7 = q_t_1_8 * e_dg_8; + float kp_19 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_8) * kdg_0[7]); + float dqf_8 = pool_f32[ew_2 + 896 + d_lane]; + float dkf_8 = pool_f32[ew_2 + 1920 + d_lane]; + float dgf_8 = pool_f32[ew_2 + 2944 + d_lane]; + float dq_o_8 = dq2_8 + dqf_8; + float dk_o_8 = dk2b_8 + dkf_8 + dkt_3[7]; + float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_3[7], k_t_8, cr_7)) + dgf_8; + s_run_2[0] = s_run_2[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_24 = t_24 * qrs; - if (t_24 < clen) { + int qoff_8 = t_8 * qrs; + if (t_8 < clen) { { - auto _gst_487 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_487)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_487) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_491 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_491)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_488 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_488)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_488) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_492 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_492)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_8; } } - int t_25 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_25 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_25 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_25 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_25 >= clen) ? 0.0f : q_t_25); - float dqf_25 = pool_f32[ew_2 + 768 + d_lane]; - float dkf_25 = pool_f32[ew_2 + 1792 + d_lane]; - float dgf_25 = pool_f32[ew_2 + 2816 + d_lane]; - float bt_t_25 = btv_2[6]; - float e_off_25 = approx_exp2(g_t_25 - gn_2); - float e_dg_25 = approx_exp2(g_t_25 - gc_3); - float dq2_25 = qdg_2[6] * e_dg_25; - float dk2_25 = kdg_2[6] * e_dg_25; - dbp_2[6] = dk2_25 * k_t_25; - float dk2b_25 = dk2_25 * bt_t_25; - float dq_o_25 = dq2_25 + dqf_25; - float dk_o_25 = dk2b_25 + dkf_25 + dkt_3[6]; - float dg_o_25 = __fmaf_rn(q_t_1_25, dq2_25, (dk2b_25 - dkt_3[6]) * k_t_25) + dgf_25; - s_run_2[0] = s_run_2[0] + dg_o_25; + int t_9 = 8 * th + 6; + float r_284[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_9 = r_284[0]; + unsigned int r_285[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_9 = __uint_as_float(r_285[0] << 16); + unsigned int r_286[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_9 = __uint_as_float(r_286[0] << 16); + float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); + float bt_t_9 = btv_0[6]; + float e_off_9 = approx_exp2(g_t_9 - gn_2); + float e_dg_9 = approx_exp2(g_t_9 - gc_3); + float dq2_9 = qdg_0[6] * e_dg_9; + float dk2_9 = kdg_0[6] * e_dg_9; + dbp_2[6] = dk2_9 * k_t_9; + float dk2b_9 = dk2_9 * bt_t_9; + float qe_8 = q_t_1_9 * e_dg_9; + float kp_20 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_9) * kdg_0[6]); + float dqf_9 = pool_f32[ew_2 + 768 + d_lane]; + float dkf_9 = pool_f32[ew_2 + 1792 + d_lane]; + float dgf_9 = pool_f32[ew_2 + 2816 + d_lane]; + float dq_o_9 = dq2_9 + dqf_9; + float dk_o_9 = dk2b_9 + dkf_9 + dkt_3[6]; + float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_3[6], k_t_9, cr_8)) + dgf_9; + s_run_2[0] = s_run_2[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_25 = t_25 * qrs; - if (t_25 < clen) { + int qoff_9 = t_9 * qrs; + if (t_9 < clen) { { - auto _gst_489 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_489)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_489) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_493 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_493)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_490 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_490)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_490) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_494 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_494)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_9; } } - int t_26 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_26 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_26 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_26 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_26 >= clen) ? 0.0f : q_t_26); - float dqf_26 = pool_f32[ew_2 + 640 + d_lane]; - float dkf_26 = pool_f32[ew_2 + 1664 + d_lane]; - float dgf_26 = pool_f32[ew_2 + 2688 + d_lane]; - float bt_t_26 = btv_2[5]; - float e_off_26 = approx_exp2(g_t_26 - gn_2); - float e_dg_26 = approx_exp2(g_t_26 - gc_3); - float dq2_26 = qdg_2[5] * e_dg_26; - float dk2_26 = kdg_2[5] * e_dg_26; - dbp_2[5] = dk2_26 * k_t_26; - float dk2b_26 = dk2_26 * bt_t_26; - float dq_o_26 = dq2_26 + dqf_26; - float dk_o_26 = dk2b_26 + dkf_26 + dkt_3[5]; - float dg_o_26 = __fmaf_rn(q_t_1_26, dq2_26, (dk2b_26 - dkt_3[5]) * k_t_26) + dgf_26; - s_run_2[0] = s_run_2[0] + dg_o_26; + int t_10 = 8 * th + 5; + float r_287[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_10 = r_287[0]; + unsigned int r_288[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_10 = __uint_as_float(r_288[0] << 16); + unsigned int r_289[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_10 = __uint_as_float(r_289[0] << 16); + float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); + float bt_t_10 = btv_0[5]; + float e_off_10 = approx_exp2(g_t_10 - gn_2); + float e_dg_10 = approx_exp2(g_t_10 - gc_3); + float dq2_10 = qdg_0[5] * e_dg_10; + float dk2_10 = kdg_0[5] * e_dg_10; + dbp_2[5] = dk2_10 * k_t_10; + float dk2b_10 = dk2_10 * bt_t_10; + float qe_9 = q_t_1_10 * e_dg_10; + float kp_21 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_10) * kdg_0[5]); + float dqf_10 = pool_f32[ew_2 + 640 + d_lane]; + float dkf_10 = pool_f32[ew_2 + 1664 + d_lane]; + float dgf_10 = pool_f32[ew_2 + 2688 + d_lane]; + float dq_o_10 = dq2_10 + dqf_10; + float dk_o_10 = dk2b_10 + dkf_10 + dkt_3[5]; + float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_3[5], k_t_10, cr_9)) + dgf_10; + s_run_2[0] = s_run_2[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_26 = t_26 * qrs; - if (t_26 < clen) { + int qoff_10 = t_10 * qrs; + if (t_10 < clen) { { - auto _gst_491 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_491)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_495 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_495)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_492 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_492)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_496 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_496)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_10; } } - int t_27 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_27 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_27 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_27 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_27 >= clen) ? 0.0f : q_t_27); - float dqf_27 = pool_f32[ew_2 + 512 + d_lane]; - float dkf_27 = pool_f32[ew_2 + 1536 + d_lane]; - float dgf_27 = pool_f32[ew_2 + 2560 + d_lane]; - float bt_t_27 = btv_2[4]; - float e_off_27 = approx_exp2(g_t_27 - gn_2); - float e_dg_27 = approx_exp2(g_t_27 - gc_3); - float dq2_27 = qdg_2[4] * e_dg_27; - float dk2_27 = kdg_2[4] * e_dg_27; - dbp_2[4] = dk2_27 * k_t_27; - float dk2b_27 = dk2_27 * bt_t_27; - float dq_o_27 = dq2_27 + dqf_27; - float dk_o_27 = dk2b_27 + dkf_27 + dkt_3[4]; - float dg_o_27 = __fmaf_rn(q_t_1_27, dq2_27, (dk2b_27 - dkt_3[4]) * k_t_27) + dgf_27; - s_run_2[0] = s_run_2[0] + dg_o_27; + int t_11 = 8 * th + 4; + float r_290[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_11 = r_290[0]; + unsigned int r_291[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_11 = __uint_as_float(r_291[0] << 16); + unsigned int r_292[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_11 = __uint_as_float(r_292[0] << 16); + float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); + float bt_t_11 = btv_0[4]; + float e_off_11 = approx_exp2(g_t_11 - gn_2); + float e_dg_11 = approx_exp2(g_t_11 - gc_3); + float dq2_11 = qdg_0[4] * e_dg_11; + float dk2_11 = kdg_0[4] * e_dg_11; + dbp_2[4] = dk2_11 * k_t_11; + float dk2b_11 = dk2_11 * bt_t_11; + float qe_10 = q_t_1_11 * e_dg_11; + float kp_22 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_11) * kdg_0[4]); + float dqf_11 = pool_f32[ew_2 + 512 + d_lane]; + float dkf_11 = pool_f32[ew_2 + 1536 + d_lane]; + float dgf_11 = pool_f32[ew_2 + 2560 + d_lane]; + float dq_o_11 = dq2_11 + dqf_11; + float dk_o_11 = dk2b_11 + dkf_11 + dkt_3[4]; + float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_3[4], k_t_11, cr_10)) + dgf_11; + s_run_2[0] = s_run_2[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_27 = t_27 * qrs; - if (t_27 < clen) { + int qoff_11 = t_11 * qrs; + if (t_11 < clen) { { - auto _gst_493 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_493)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_497 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_497)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_494 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_494)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_498 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_498)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_11; } } - int t_28 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_28 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_28 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_28 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_28 >= clen) ? 0.0f : q_t_28); - float dqf_28 = pool_f32[ew_2 + 384 + d_lane]; - float dkf_28 = pool_f32[ew_2 + 1408 + d_lane]; - float dgf_28 = pool_f32[ew_2 + 2432 + d_lane]; - float bt_t_28 = btv_2[3]; - float e_off_28 = approx_exp2(g_t_28 - gn_2); - float e_dg_28 = approx_exp2(g_t_28 - gc_3); - float dq2_28 = qdg_2[3] * e_dg_28; - float dk2_28 = kdg_2[3] * e_dg_28; - dbp_2[3] = dk2_28 * k_t_28; - float dk2b_28 = dk2_28 * bt_t_28; - float dq_o_28 = dq2_28 + dqf_28; - float dk_o_28 = dk2b_28 + dkf_28 + dkt_3[3]; - float dg_o_28 = __fmaf_rn(q_t_1_28, dq2_28, (dk2b_28 - dkt_3[3]) * k_t_28) + dgf_28; - s_run_2[0] = s_run_2[0] + dg_o_28; + int t_12 = 8 * th + 3; + float r_293[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_12 = r_293[0]; + unsigned int r_294[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_12 = __uint_as_float(r_294[0] << 16); + unsigned int r_295[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_12 = __uint_as_float(r_295[0] << 16); + float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); + float bt_t_12 = btv_0[3]; + float e_off_12 = approx_exp2(g_t_12 - gn_2); + float e_dg_12 = approx_exp2(g_t_12 - gc_3); + float dq2_12 = qdg_0[3] * e_dg_12; + float dk2_12 = kdg_0[3] * e_dg_12; + dbp_2[3] = dk2_12 * k_t_12; + float dk2b_12 = dk2_12 * bt_t_12; + float qe_11 = q_t_1_12 * e_dg_12; + float kp_23 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_12) * kdg_0[3]); + float dqf_12 = pool_f32[ew_2 + 384 + d_lane]; + float dkf_12 = pool_f32[ew_2 + 1408 + d_lane]; + float dgf_12 = pool_f32[ew_2 + 2432 + d_lane]; + float dq_o_12 = dq2_12 + dqf_12; + float dk_o_12 = dk2b_12 + dkf_12 + dkt_3[3]; + float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_3[3], k_t_12, cr_11)) + dgf_12; + s_run_2[0] = s_run_2[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_28 = t_28 * qrs; - if (t_28 < clen) { + int qoff_12 = t_12 * qrs; + if (t_12 < clen) { { - auto _gst_495 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_495)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_499 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_499)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_496 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_496)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_500 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_500)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_12; } } - int t_29 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_29 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_29 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_29 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_29 >= clen) ? 0.0f : q_t_29); - float dqf_29 = pool_f32[ew_2 + 256 + d_lane]; - float dkf_29 = pool_f32[ew_2 + 1280 + d_lane]; - float dgf_29 = pool_f32[ew_2 + 2304 + d_lane]; - float bt_t_29 = btv_2[2]; - float e_off_29 = approx_exp2(g_t_29 - gn_2); - float e_dg_29 = approx_exp2(g_t_29 - gc_3); - float dq2_29 = qdg_2[2] * e_dg_29; - float dk2_29 = kdg_2[2] * e_dg_29; - dbp_2[2] = dk2_29 * k_t_29; - float dk2b_29 = dk2_29 * bt_t_29; - float dq_o_29 = dq2_29 + dqf_29; - float dk_o_29 = dk2b_29 + dkf_29 + dkt_3[2]; - float dg_o_29 = __fmaf_rn(q_t_1_29, dq2_29, (dk2b_29 - dkt_3[2]) * k_t_29) + dgf_29; - s_run_2[0] = s_run_2[0] + dg_o_29; + int t_13 = 8 * th + 2; + float r_296[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_13 = r_296[0]; + unsigned int r_297[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_13 = __uint_as_float(r_297[0] << 16); + unsigned int r_298[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_13 = __uint_as_float(r_298[0] << 16); + float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); + float bt_t_13 = btv_0[2]; + float e_off_13 = approx_exp2(g_t_13 - gn_2); + float e_dg_13 = approx_exp2(g_t_13 - gc_3); + float dq2_13 = qdg_0[2] * e_dg_13; + float dk2_13 = kdg_0[2] * e_dg_13; + dbp_2[2] = dk2_13 * k_t_13; + float dk2b_13 = dk2_13 * bt_t_13; + float qe_12 = q_t_1_13 * e_dg_13; + float kp_24 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_13) * kdg_0[2]); + float dqf_13 = pool_f32[ew_2 + 256 + d_lane]; + float dkf_13 = pool_f32[ew_2 + 1280 + d_lane]; + float dgf_13 = pool_f32[ew_2 + 2304 + d_lane]; + float dq_o_13 = dq2_13 + dqf_13; + float dk_o_13 = dk2b_13 + dkf_13 + dkt_3[2]; + float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_3[2], k_t_13, cr_12)) + dgf_13; + s_run_2[0] = s_run_2[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_29 = t_29 * qrs; - if (t_29 < clen) { + int qoff_13 = t_13 * qrs; + if (t_13 < clen) { { - auto _gst_497 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_497)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_501 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_501)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_498 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_498)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_502 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_502)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_13; } } - int t_30 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_30 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_30 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_30 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_30 >= clen) ? 0.0f : q_t_30); - float dqf_30 = pool_f32[ew_2 + 128 + d_lane]; - float dkf_30 = pool_f32[ew_2 + 1152 + d_lane]; - float dgf_30 = pool_f32[ew_2 + 2176 + d_lane]; - float bt_t_30 = btv_2[1]; - float e_off_30 = approx_exp2(g_t_30 - gn_2); - float e_dg_30 = approx_exp2(g_t_30 - gc_3); - float dq2_30 = qdg_2[1] * e_dg_30; - float dk2_30 = kdg_2[1] * e_dg_30; - dbp_2[1] = dk2_30 * k_t_30; - float dk2b_30 = dk2_30 * bt_t_30; - float dq_o_30 = dq2_30 + dqf_30; - float dk_o_30 = dk2b_30 + dkf_30 + dkt_3[1]; - float dg_o_30 = __fmaf_rn(q_t_1_30, dq2_30, (dk2b_30 - dkt_3[1]) * k_t_30) + dgf_30; - s_run_2[0] = s_run_2[0] + dg_o_30; + int t_14 = 8 * th + 1; + float r_299[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_14 = r_299[0]; + unsigned int r_300[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_14 = __uint_as_float(r_300[0] << 16); + unsigned int r_301[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_14 = __uint_as_float(r_301[0] << 16); + float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); + float bt_t_14 = btv_0[1]; + float e_off_14 = approx_exp2(g_t_14 - gn_2); + float e_dg_14 = approx_exp2(g_t_14 - gc_3); + float dq2_14 = qdg_0[1] * e_dg_14; + float dk2_14 = kdg_0[1] * e_dg_14; + dbp_2[1] = dk2_14 * k_t_14; + float dk2b_14 = dk2_14 * bt_t_14; + float qe_13 = q_t_1_14 * e_dg_14; + float kp_25 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_14) * kdg_0[1]); + float dqf_14 = pool_f32[ew_2 + 128 + d_lane]; + float dkf_14 = pool_f32[ew_2 + 1152 + d_lane]; + float dgf_14 = pool_f32[ew_2 + 2176 + d_lane]; + float dq_o_14 = dq2_14 + dqf_14; + float dk_o_14 = dk2b_14 + dkf_14 + dkt_3[1]; + float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_3[1], k_t_14, cr_13)) + dgf_14; + s_run_2[0] = s_run_2[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_30 = t_30 * qrs; - if (t_30 < clen) { + int qoff_14 = t_14 * qrs; + if (t_14 < clen) { { - auto _gst_499 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_499)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_503 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_503)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_503) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_500 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_500)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_504 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_504)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_504) + (0)) = (__nv_bfloat16)dk_o_14; } } - int t_31 = 0 + 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_31 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_31 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_31 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_31 >= clen) ? 0.0f : q_t_31); - float dqf_31 = pool_f32[ew_2 + d_lane]; - float dkf_31 = pool_f32[ew_2 + 1024 + d_lane]; - float dgf_31 = pool_f32[ew_2 + 2048 + d_lane]; - float bt_t_31 = btv_2[0]; - float e_off_31 = approx_exp2(g_t_31 - gn_2); - float e_dg_31 = approx_exp2(g_t_31 - gc_3); - float dq2_31 = qdg_2[0] * e_dg_31; - float dk2_31 = kdg_2[0] * e_dg_31; - dbp_2[0] = dk2_31 * k_t_31; - float dk2b_31 = dk2_31 * bt_t_31; - float dq_o_31 = dq2_31 + dqf_31; - float dk_o_31 = dk2b_31 + dkf_31 + dkt_3[0]; - float dg_o_31 = __fmaf_rn(q_t_1_31, dq2_31, (dk2b_31 - dkt_3[0]) * k_t_31) + dgf_31; - s_run_2[0] = s_run_2[0] + dg_o_31; + int t_15 = 0 + 8 * th; + float r_302[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_15 = r_302[0]; + unsigned int r_303[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_15 = __uint_as_float(r_303[0] << 16); + unsigned int r_304[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_15 = __uint_as_float(r_304[0] << 16); + float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); + float bt_t_15 = btv_0[0]; + float e_off_15 = approx_exp2(g_t_15 - gn_2); + float e_dg_15 = approx_exp2(g_t_15 - gc_3); + float dq2_15 = qdg_0[0] * e_dg_15; + float dk2_15 = kdg_0[0] * e_dg_15; + dbp_2[0] = dk2_15 * k_t_15; + float dk2b_15 = dk2_15 * bt_t_15; + float qe_14 = q_t_1_15 * e_dg_15; + float kp_26 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_15) * kdg_0[0]); + float dqf_15 = pool_f32[ew_2 + d_lane]; + float dkf_15 = pool_f32[ew_2 + 1024 + d_lane]; + float dgf_15 = pool_f32[ew_2 + 2048 + d_lane]; + float dq_o_15 = dq2_15 + dqf_15; + float dk_o_15 = dk2b_15 + dkf_15 + dkt_3[0]; + float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_3[0], k_t_15, cr_14)) + dgf_15; + s_run_2[0] = s_run_2[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_31 = t_31 * qrs; - if (t_31 < clen) { + int qoff_15 = t_15 * qrs; + if (t_15 < clen) { { - auto _gst_501 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_501)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_505 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_505)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_505) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_502 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_502)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_506 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_506)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_506) + (0)) = (__nv_bfloat16)dk_o_15; } } float tot_2 = s_run_2[0]; @@ -5626,37 +5281,37 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_503 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_503; + float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_507; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_504; + float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_508; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_505 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_505; + float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_509; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_506; + float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_510; float v2_18[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_18[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_507; + float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_18[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_511; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_18[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_508; + float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_18[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_512; float send_26 = ((b2_2 == 0) ? v2_18[1] : v2_18[0]); - float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_18[0] : v2_18[1]) + _shfl_xor_sync_509; - float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_510; - float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_511; + float _shfl_xor_sync_513 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_18[0] : v2_18[1]) + _shfl_xor_sync_513; + float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_514; + float _shfl_xor_sync_515 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_515; tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_512 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_512), "f"(tot_2) : "memory"); + uint32_t _addr_516 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_516), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5688,132 +5343,132 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_513 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_517 = make_float2(gacc[4], gacc[4]); float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_513), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_516; - asm("abs.f32 %0, %1;" : "=f"(_abs_516) : "f"(rp_z_11.x)); - float _neg_515; - asm("neg.f32 %0, %1;" : "=f"(_neg_515) : "f"(_abs_516)); - float _exp_514 = __nv_expf(_neg_515); - float _abs_519; - asm("abs.f32 %0, %1;" : "=f"(_abs_519) : "f"(rp_z_11.y)); - float _neg_518; - asm("neg.f32 %0, %1;" : "=f"(_neg_518) : "f"(_abs_519)); - float _exp_517 = __nv_expf(_neg_518); - float2 rp_e_11 = make_float2(_exp_514, _exp_517); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_517), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_520; + asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_11.x)); + float _neg_519; + asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); + float _exp_518 = __nv_expf(_neg_519); + float _abs_523; + asm("abs.f32 %0, %1;" : "=f"(_abs_523) : "f"(rp_z_11.y)); + float _neg_522; + asm("neg.f32 %0, %1;" : "=f"(_neg_522) : "f"(_abs_523)); + float _exp_521 = __nv_expf(_neg_522); + float2 rp_e_11 = make_float2(_exp_518, _exp_521); float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_520 = approx_rcp(rp_den_11.x); - float _rcp_521 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_522 = make_float2(_rcp_520, _rcp_521); + float _rcp_524 = approx_rcp(rp_den_11.x); + float _rcp_525 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_526 = make_float2(_rcp_524, _rcp_525); float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_522)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_526)); float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_523 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_527 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_527)); u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_524 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_528 = make_float2(gacc[4], gacc[4]); float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_524), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_527; - asm("abs.f32 %0, %1;" : "=f"(_abs_527) : "f"(rp_z_12.x)); - float _neg_526; - asm("neg.f32 %0, %1;" : "=f"(_neg_526) : "f"(_abs_527)); - float _exp_525 = __nv_expf(_neg_526); - float _abs_530; - asm("abs.f32 %0, %1;" : "=f"(_abs_530) : "f"(rp_z_12.y)); - float _neg_529; - asm("neg.f32 %0, %1;" : "=f"(_neg_529) : "f"(_abs_530)); - float _exp_528 = __nv_expf(_neg_529); - float2 rp_e_12 = make_float2(_exp_525, _exp_528); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_528), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_531; + asm("abs.f32 %0, %1;" : "=f"(_abs_531) : "f"(rp_z_12.x)); + float _neg_530; + asm("neg.f32 %0, %1;" : "=f"(_neg_530) : "f"(_abs_531)); + float _exp_529 = __nv_expf(_neg_530); + float _abs_534; + asm("abs.f32 %0, %1;" : "=f"(_abs_534) : "f"(rp_z_12.y)); + float _neg_533; + asm("neg.f32 %0, %1;" : "=f"(_neg_533) : "f"(_abs_534)); + float _exp_532 = __nv_expf(_neg_533); + float2 rp_e_12 = make_float2(_exp_529, _exp_532); float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_531 = approx_rcp(rp_den_12.x); - float _rcp_532 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_533 = make_float2(_rcp_531, _rcp_532); + float _rcp_535 = approx_rcp(rp_den_12.x); + float _rcp_536 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_537 = make_float2(_rcp_535, _rcp_536); float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_533)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_537)); float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_534 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_538 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_534)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_538)); u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_535 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_539 = make_float2(gacc[4], gacc[4]); float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_535), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_538; - asm("abs.f32 %0, %1;" : "=f"(_abs_538) : "f"(rp_z_13.x)); - float _neg_537; - asm("neg.f32 %0, %1;" : "=f"(_neg_537) : "f"(_abs_538)); - float _exp_536 = __nv_expf(_neg_537); - float _abs_541; - asm("abs.f32 %0, %1;" : "=f"(_abs_541) : "f"(rp_z_13.y)); - float _neg_540; - asm("neg.f32 %0, %1;" : "=f"(_neg_540) : "f"(_abs_541)); - float _exp_539 = __nv_expf(_neg_540); - float2 rp_e_13 = make_float2(_exp_536, _exp_539); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_539), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_542; + asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(rp_z_13.x)); + float _neg_541; + asm("neg.f32 %0, %1;" : "=f"(_neg_541) : "f"(_abs_542)); + float _exp_540 = __nv_expf(_neg_541); + float _abs_545; + asm("abs.f32 %0, %1;" : "=f"(_abs_545) : "f"(rp_z_13.y)); + float _neg_544; + asm("neg.f32 %0, %1;" : "=f"(_neg_544) : "f"(_abs_545)); + float _exp_543 = __nv_expf(_neg_544); + float2 rp_e_13 = make_float2(_exp_540, _exp_543); float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_542 = approx_rcp(rp_den_13.x); - float _rcp_543 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_544 = make_float2(_rcp_542, _rcp_543); + float _rcp_546 = approx_rcp(rp_den_13.x); + float _rcp_547 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_548 = make_float2(_rcp_546, _rcp_547); float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_544)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_548)); float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_545 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_549 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_545)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_549)); u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_546 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_550 = make_float2(gacc[4], gacc[4]); float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_546), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_549; - asm("abs.f32 %0, %1;" : "=f"(_abs_549) : "f"(rp_z_14.x)); - float _neg_548; - asm("neg.f32 %0, %1;" : "=f"(_neg_548) : "f"(_abs_549)); - float _exp_547 = __nv_expf(_neg_548); - float _abs_552; - asm("abs.f32 %0, %1;" : "=f"(_abs_552) : "f"(rp_z_14.y)); - float _neg_551; - asm("neg.f32 %0, %1;" : "=f"(_neg_551) : "f"(_abs_552)); - float _exp_550 = __nv_expf(_neg_551); - float2 rp_e_14 = make_float2(_exp_547, _exp_550); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_550), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_553; + asm("abs.f32 %0, %1;" : "=f"(_abs_553) : "f"(rp_z_14.x)); + float _neg_552; + asm("neg.f32 %0, %1;" : "=f"(_neg_552) : "f"(_abs_553)); + float _exp_551 = __nv_expf(_neg_552); + float _abs_556; + asm("abs.f32 %0, %1;" : "=f"(_abs_556) : "f"(rp_z_14.y)); + float _neg_555; + asm("neg.f32 %0, %1;" : "=f"(_neg_555) : "f"(_abs_556)); + float _exp_554 = __nv_expf(_neg_555); + float2 rp_e_14 = make_float2(_exp_551, _exp_554); float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_553 = approx_rcp(rp_den_14.x); - float _rcp_554 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_555 = make_float2(_rcp_553, _rcp_554); + float _rcp_557 = approx_rcp(rp_den_14.x); + float _rcp_558 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_559 = make_float2(_rcp_557, _rcp_558); float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_555)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_559)); float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_556 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_560 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_556)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_560)); u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -5828,54 +5483,54 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_557; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_557) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_557; + uint32_t _prmt_561; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_561; int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_558 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_558)); - *(reinterpret_cast(_gst_558) + (0)) = pq_12[0]; + auto _gst_562 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_562)); + *(reinterpret_cast(_gst_562) + (0)) = pq_12[0]; } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_559; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_559) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_559; + uint32_t _prmt_563; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_563; int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_560 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_560)); - *(reinterpret_cast(_gst_560) + (0)) = pq_13[0]; + auto _gst_564 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_564)); + *(reinterpret_cast(_gst_564) + (0)) = pq_13[0]; } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_561; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_561; + uint32_t _prmt_565; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_565) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_565; int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_562 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_562)); - *(reinterpret_cast(_gst_562) + (0)) = pq_14[0]; + auto _gst_566 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_566)); + *(reinterpret_cast(_gst_566) + (0)) = pq_14[0]; } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_563; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_563; + uint32_t _prmt_567; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_567) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_567; int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_564 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_564)); - *(reinterpret_cast(_gst_564) + (0)) = pq_15[0]; + auto _gst_568 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_568)); + *(reinterpret_cast(_gst_568) + (0)) = pq_15[0]; } } int t_a_24 = 8 * th; @@ -5940,47 +5595,47 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_565; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_565) : "f"(s_a_2), "f"(nm_t_5)); + float _sub_569; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_569) : "f"(s_a_2), "f"(nm_t_5)); float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_565), "f"(pa_2)); - float _sub_566; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_566) : "f"(pa_2), "f"(nm_t_5)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_569), "f"(pa_2)); + float _sub_570; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_570) : "f"(pa_2), "f"(nm_t_5)); float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_566), "f"(s_a_2)); - float _abs_568; - asm("abs.f32 %0, %1;" : "=f"(_abs_568) : "f"(s_a_2)); - float _abs_569; - asm("abs.f32 %0, %1;" : "=f"(_abs_569) : "f"(pa_2)); - uint32_t _setp_567_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_567_bits) : "f"(_abs_568), "f"(_abs_569)); - bool _setp_567 = _setp_567_bits != 0; - float nm_err_5 = ((_setp_567) ? nm_es_5 : nm_ex_5); - float _add_570; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_570; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_570), "f"(s_a_2)); + float _abs_572; + asm("abs.f32 %0, %1;" : "=f"(_abs_572) : "f"(s_a_2)); + float _abs_573; + asm("abs.f32 %0, %1;" : "=f"(_abs_573) : "f"(pa_2)); + uint32_t _setp_571_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_571_bits) : "f"(_abs_572), "f"(_abs_573)); + bool _setp_571 = _setp_571_bits != 0; + float nm_err_5 = ((_setp_571) ? nm_es_5 : nm_ex_5); + float _add_574; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_574; s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_571; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(s_b_2), "f"(nm_t_6)); + float _sub_575; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_575) : "f"(s_b_2), "f"(nm_t_6)); float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_571), "f"(pb_2)); - float _sub_572; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_572) : "f"(pb_2), "f"(nm_t_6)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_575), "f"(pb_2)); + float _sub_576; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_576) : "f"(pb_2), "f"(nm_t_6)); float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_572), "f"(s_b_2)); - float _abs_574; - asm("abs.f32 %0, %1;" : "=f"(_abs_574) : "f"(s_b_2)); - float _abs_575; - asm("abs.f32 %0, %1;" : "=f"(_abs_575) : "f"(pb_2)); - uint32_t _setp_573_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_573_bits) : "f"(_abs_574), "f"(_abs_575)); - bool _setp_573 = _setp_573_bits != 0; - float nm_err_6 = ((_setp_573) ? nm_es_6 : nm_ex_6); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_576; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_576), "f"(s_b_2)); + float _abs_578; + asm("abs.f32 %0, %1;" : "=f"(_abs_578) : "f"(s_b_2)); + float _abs_579; + asm("abs.f32 %0, %1;" : "=f"(_abs_579) : "f"(pb_2)); + uint32_t _setp_577_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_577_bits) : "f"(_abs_578), "f"(_abs_579)); + bool _setp_577 = _setp_577_bits != 0; + float nm_err_6 = ((_setp_577) ? nm_es_6 : nm_ex_6); + float _add_580; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_580; s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -6019,60 +5674,60 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_578; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_578) : "f"(ts_s), "f"(ts_bb)); - float _sub_577; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_577) : "f"(s_a_3), "f"(_sub_578)); - float _sub_579; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_579) : "f"(other[0]), "f"(ts_bb)); + float _sub_582; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s), "f"(ts_bb)); + float _sub_581; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(s_a_3), "f"(_sub_582)); + float _sub_583; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(other[0]), "f"(ts_bb)); float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_577), "f"(_sub_579)); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_a_3), "f"(other[1])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_581), "f"(_sub_583)); + float _add_584; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(c_a_3), "f"(other[1])); float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_580), "f"(ts_e)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_584), "f"(ts_e)); float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_582; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_581; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(ts_s), "f"(_sub_582)); - float _sub_583; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(pm_c), "f"(ts_bb_0)); + float _sub_586; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_585; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s), "f"(_sub_586)); + float _sub_587; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_587) : "f"(pm_c), "f"(ts_bb_0)); float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_581), "f"(_sub_583)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_585), "f"(_sub_587)); s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_585; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_584; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_584) : "f"(s_b_3), "f"(_sub_585)); - float _sub_586; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(other[2]), "f"(ts_bb_1)); + float _sub_589; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_588; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(s_b_3), "f"(_sub_589)); + float _sub_590; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(other[2]), "f"(ts_bb_1)); float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_584), "f"(_sub_586)); - float _add_587; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_587) : "f"(c_b_3), "f"(other[3])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_588), "f"(_sub_590)); + float _add_591; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_591) : "f"(c_b_3), "f"(other[3])); float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_587), "f"(ts_e_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_591), "f"(ts_e_1)); float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_589; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_588; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(ts_s_1), "f"(_sub_589)); - float _sub_590; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(pm_c_0), "f"(ts_bb_2)); + float _sub_593; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_593) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_592; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_592) : "f"(ts_s_1), "f"(_sub_593)); + float _sub_594; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_594) : "f"(pm_c_0), "f"(ts_bb_2)); float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_588), "f"(_sub_590)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_592), "f"(_sub_594)); s_b_3 = ts_s_2; c_b_3 = ts_e_2; double v_a = (double)s_a_3 + (double)c_a_3; @@ -6175,41 +5830,41 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + for (int r_305 = 0; r_305 < 16; r_305++) { + int t_16 = r_305; + if (t_16 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6276,41 +5931,41 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_32 = 16 + r_348; - if (t_32 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + for (int r_306 = 0; r_306 < 16; r_306++) { + int t_17 = 16 + r_306; + if (t_17 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6416,41 +6071,41 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_33 = 32 + r_349; - if (t_33 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + for (int r_307 = 0; r_307 < 16; r_307++) { + int t_18 = 32 + r_307; + if (t_18 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6583,41 +6238,41 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_34 = 48 + r_350; - if (t_34 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + for (int r_308 = 0; r_308 < 16; r_308++) { + int t_19 = 48 + r_308; + if (t_19 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu index 4a94bfa903d..ac525b966de 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu @@ -130,6 +130,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -172,6 +179,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2235,15 +2258,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2264,15 +2293,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2293,15 +2328,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2322,15 +2363,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2366,12 +2413,18 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2391,12 +2444,18 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2416,12 +2475,18 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2441,12 +2506,18 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2491,15 +2562,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2520,15 +2597,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2549,15 +2632,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2578,15 +2667,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2625,15 +2720,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2654,15 +2755,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2683,15 +2790,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2712,15 +2825,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2791,37 +2910,37 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2833,29 +2952,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2863,29 +2982,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2953,24 +3072,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; - float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0) + dgf_0; + float qe = q_t0_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr)) + dgf_0; int goff_0 = t_0 * rs_g; int qoff_0 = t_0 * qrs; { - auto _gst_216 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast(_gst_216) + (0)) = dg_o_0; + auto _gst_248 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast(_gst_248) + (0)) = dg_o_0; } if (t_0 < clen) { { - auto _gst_217 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_249 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_218 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_250 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -2996,24 +3120,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_1 = dk2_1 * bt_t_1; float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; - float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1) + dgf_1; + float qe_0 = q_t0_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0)) + dgf_1; int goff_1 = t_1 * rs_g; int qoff_1 = t_1 * qrs; { - auto _gst_219 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast(_gst_219) + (0)) = dg_o_1; + auto _gst_251 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast(_gst_251) + (0)) = dg_o_1; } if (t_1 < clen) { { - auto _gst_220 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_252 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_221 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_253 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3039,24 +3168,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; - float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2) + dgf_2; + float qe_1 = q_t0_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1)) + dgf_2; int goff_2 = t_2 * rs_g; int qoff_2 = t_2 * qrs; { - auto _gst_222 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast(_gst_222) + (0)) = dg_o_2; + auto _gst_254 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast(_gst_254) + (0)) = dg_o_2; } if (t_2 < clen) { { - auto _gst_223 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_255 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_224 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_256 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3082,24 +3216,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; - float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3) + dgf_3; + float qe_2 = q_t0_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2)) + dgf_3; int goff_3 = t_3 * rs_g; int qoff_3 = t_3 * qrs; { - auto _gst_225 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast(_gst_225) + (0)) = dg_o_3; + auto _gst_257 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast(_gst_257) + (0)) = dg_o_3; } if (t_3 < clen) { { - auto _gst_226 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_258 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_227 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_259 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3125,24 +3264,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; - float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4) + dgf_4; + float qe_3 = q_t0_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3)) + dgf_4; int goff_4 = t_4 * rs_g; int qoff_4 = t_4 * qrs; { - auto _gst_228 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast(_gst_228) + (0)) = dg_o_4; + auto _gst_260 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast(_gst_260) + (0)) = dg_o_4; } if (t_4 < clen) { { - auto _gst_229 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_261 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_230 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_262 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3168,24 +3312,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; - float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5) + dgf_5; + float qe_4 = q_t0_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4)) + dgf_5; int goff_5 = t_5 * rs_g; int qoff_5 = t_5 * qrs; { - auto _gst_231 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast(_gst_231) + (0)) = dg_o_5; + auto _gst_263 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast(_gst_263) + (0)) = dg_o_5; } if (t_5 < clen) { { - auto _gst_232 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_232)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_232) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_264 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_264)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_264) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_233 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_233)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_233) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_265 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_265)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3211,24 +3360,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; - float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6) + dgf_6; + float qe_5 = q_t0_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5)) + dgf_6; int goff_6 = t_6 * rs_g; int qoff_6 = t_6 * qrs; { - auto _gst_234 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_234)); - *(reinterpret_cast(_gst_234) + (0)) = dg_o_6; + auto _gst_266 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_266)); + *(reinterpret_cast(_gst_266) + (0)) = dg_o_6; } if (t_6 < clen) { { - auto _gst_235 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_235)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_235) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_267 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_267)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_267) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_236 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_236)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_236) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_268 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_268)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3254,24 +3408,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; - float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7) + dgf_7; + float qe_6 = q_t0_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6)) + dgf_7; int goff_7 = t_7 * rs_g; int qoff_7 = t_7 * qrs; { - auto _gst_237 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_237)); - *(reinterpret_cast(_gst_237) + (0)) = dg_o_7; + auto _gst_269 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_269)); + *(reinterpret_cast(_gst_269) + (0)) = dg_o_7; } if (t_7 < clen) { { - auto _gst_238 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_238)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_238) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_270 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_270)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_270) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_239 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_239)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_239) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_271 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_271)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dk_o_7; } } int b4 = lane_1 >> 4 & 1; @@ -3279,31 +3438,31 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_240; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_272; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_241 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_241; + float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_273; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_242 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_242; + float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_274; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_243 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_243; + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_275; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_244 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_244; + float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_276; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_245 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_245; + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_277; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_246 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_246; - float _shfl_xor_sync_247 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_247; - float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_248; + float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_278; + float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_279; + float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_280; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3361,24 +3520,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; - float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8) + dgf_8; + float qe_7 = q_t0_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7)) + dgf_8; int goff_8 = t_8 * rs_g; int qoff_8 = t_8 * qrs; { - auto _gst_249 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast(_gst_249) + (0)) = dg_o_8; + auto _gst_281 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast(_gst_281) + (0)) = dg_o_8; } if (t_8 < clen) { { - auto _gst_250 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_282 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_251 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_283 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3404,24 +3568,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; - float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9) + dgf_9; + float qe_8 = q_t0_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8)) + dgf_9; int goff_9 = t_9 * rs_g; int qoff_9 = t_9 * qrs; { - auto _gst_252 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast(_gst_252) + (0)) = dg_o_9; + auto _gst_284 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast(_gst_284) + (0)) = dg_o_9; } if (t_9 < clen) { { - auto _gst_253 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_285 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_254 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_286 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3447,24 +3616,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; - float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10) + dgf_10; + float qe_9 = q_t0_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9)) + dgf_10; int goff_10 = t_10 * rs_g; int qoff_10 = t_10 * qrs; { - auto _gst_255 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast(_gst_255) + (0)) = dg_o_10; + auto _gst_287 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast(_gst_287) + (0)) = dg_o_10; } if (t_10 < clen) { { - auto _gst_256 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_288 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_257 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_257)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_289 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_289)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3490,24 +3664,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; - float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11) + dgf_11; + float qe_10 = q_t0_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10)) + dgf_11; int goff_11 = t_11 * rs_g; int qoff_11 = t_11 * qrs; { - auto _gst_258 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_258)); - *(reinterpret_cast(_gst_258) + (0)) = dg_o_11; + auto _gst_290 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_290)); + *(reinterpret_cast(_gst_290) + (0)) = dg_o_11; } if (t_11 < clen) { { - auto _gst_259 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_259)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_291 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_291)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_260 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_260)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_292 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_292)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3533,24 +3712,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; - float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12) + dgf_12; + float qe_11 = q_t0_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11)) + dgf_12; int goff_12 = t_12 * rs_g; int qoff_12 = t_12 * qrs; { - auto _gst_261 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_261)); - *(reinterpret_cast(_gst_261) + (0)) = dg_o_12; + auto _gst_293 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_293)); + *(reinterpret_cast(_gst_293) + (0)) = dg_o_12; } if (t_12 < clen) { { - auto _gst_262 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_262)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_294 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_294)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_263 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_263)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_295 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_295)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3576,24 +3760,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; - float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13) + dgf_13; + float qe_12 = q_t0_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12)) + dgf_13; int goff_13 = t_13 * rs_g; int qoff_13 = t_13 * qrs; { - auto _gst_264 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_264)); - *(reinterpret_cast(_gst_264) + (0)) = dg_o_13; + auto _gst_296 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_296)); + *(reinterpret_cast(_gst_296) + (0)) = dg_o_13; } if (t_13 < clen) { { - auto _gst_265 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_265)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_297 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_297)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_266 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_266)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_266) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_298 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_298)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3619,24 +3808,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; - float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14) + dgf_14; + float qe_13 = q_t0_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13)) + dgf_14; int goff_14 = t_14 * rs_g; int qoff_14 = t_14 * qrs; { - auto _gst_267 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_o_14; + auto _gst_299 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_o_14; } if (t_14 < clen) { { - auto _gst_268 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_300 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_300) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_269 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_269) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_301 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3662,24 +3856,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; - float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15) + dgf_15; + float qe_14 = q_t0_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14)) + dgf_15; int goff_15 = t_15 * rs_g; int qoff_15 = t_15 * qrs; { - auto _gst_270 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_o_15; + auto _gst_302 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_o_15; } if (t_15 < clen) { { - auto _gst_271 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_303 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_303) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_272 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_272) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_304 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dk_o_15; } } int b4_0 = lane_1 >> 4 & 1; @@ -3687,31 +3886,31 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_273; + float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_305; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_274; + float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_306; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_275; + float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_307; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_276; + float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_308; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_277; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_309; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_278; + float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_310; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_279; - float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_280; - float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_281; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_311; + float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_312; + float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_313; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); mbarrier_wait(p1b_ready_addr, 0); @@ -3770,24 +3969,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; - float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16) + dgf_16; + float qe_15 = q_t0_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15)) + dgf_16; int goff_16 = t_16 * rs_g; int qoff_16 = t_16 * qrs; { - auto _gst_282 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_o_16; + auto _gst_314 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_o_16; } if (t_16 < clen) { { - auto _gst_283 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_283)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_315 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_315)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_315) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_284 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_316 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_17 = 16 + 8 * th + 1; @@ -3813,24 +4017,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; - float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17) + dgf_17; + float qe_16 = q_t0_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16)) + dgf_17; int goff_17 = t_17 * rs_g; int qoff_17 = t_17 * qrs; { - auto _gst_285 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast(_gst_285) + (0)) = dg_o_17; + auto _gst_317 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast(_gst_317) + (0)) = dg_o_17; } if (t_17 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_18 = 16 + 8 * th + 2; @@ -3856,24 +4065,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; - float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18) + dgf_18; + float qe_17 = q_t0_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17)) + dgf_18; int goff_18 = t_18 * rs_g; int qoff_18 = t_18 * qrs; { - auto _gst_288 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast(_gst_288) + (0)) = dg_o_18; + auto _gst_320 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = dg_o_18; } if (t_18 < clen) { { - auto _gst_289 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_321 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_290 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_322 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_19 = 16 + 8 * th + 3; @@ -3899,24 +4113,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; - float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19) + dgf_19; + float qe_18 = q_t0_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18)) + dgf_19; int goff_19 = t_19 * rs_g; int qoff_19 = t_19 * qrs; { - auto _gst_291 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast(_gst_291) + (0)) = dg_o_19; + auto _gst_323 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast(_gst_323) + (0)) = dg_o_19; } if (t_19 < clen) { { - auto _gst_292 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_324 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_293 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_325 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_20 = 16 + 8 * th + 4; @@ -3942,24 +4161,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; - float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20) + dgf_20; + float qe_19 = q_t0_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19)) + dgf_20; int goff_20 = t_20 * rs_g; int qoff_20 = t_20 * qrs; { - auto _gst_294 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast(_gst_294) + (0)) = dg_o_20; + auto _gst_326 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast(_gst_326) + (0)) = dg_o_20; } if (t_20 < clen) { { - auto _gst_295 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_327 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_296 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_328 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_21 = 16 + 8 * th + 5; @@ -3985,24 +4209,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; - float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21) + dgf_21; + float qe_20 = q_t0_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20)) + dgf_21; int goff_21 = t_21 * rs_g; int qoff_21 = t_21 * qrs; { - auto _gst_297 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast(_gst_297) + (0)) = dg_o_21; + auto _gst_329 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast(_gst_329) + (0)) = dg_o_21; } if (t_21 < clen) { { - auto _gst_298 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_330 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_299 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_331 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_22 = 16 + 8 * th + 6; @@ -4028,24 +4257,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; - float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22) + dgf_22; + float qe_21 = q_t0_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21)) + dgf_22; int goff_22 = t_22 * rs_g; int qoff_22 = t_22 * qrs; { - auto _gst_300 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_300)); - *(reinterpret_cast(_gst_300) + (0)) = dg_o_22; + auto _gst_332 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_332)); + *(reinterpret_cast(_gst_332) + (0)) = dg_o_22; } if (t_22 < clen) { { - auto _gst_301 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_301)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_333 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_333)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_302 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_302)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_302) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_334 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_334)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_23 = 16 + 8 * th + 7; @@ -4071,24 +4305,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; - float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23) + dgf_23; + float qe_22 = q_t0_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22)) + dgf_23; int goff_23 = t_23 * rs_g; int qoff_23 = t_23 * qrs; { - auto _gst_303 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_303)); - *(reinterpret_cast(_gst_303) + (0)) = dg_o_23; + auto _gst_335 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_335)); + *(reinterpret_cast(_gst_335) + (0)) = dg_o_23; } if (t_23 < clen) { { - auto _gst_304 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_304)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_336 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_336)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_336) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_305 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_305)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_305) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_337 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_337)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dk_o_23; } } int b4_1 = lane_1 >> 4 & 1; @@ -4096,31 +4335,31 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_306; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_338; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_307; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_339; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_308; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_340; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_309; + float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_341; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_310; + float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_342; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_311; + float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_343; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_312; - float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_313; - float _shfl_xor_sync_314 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_314; + float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_344; + float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_345; + float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_346; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); mbarrier_wait(epi_full_addr + (th) * 8, 0); @@ -4173,24 +4412,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; - float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24) + dgf_24; + float qe_23 = q_t0_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23)) + dgf_24; int goff_24 = t_24 * rs_g; int qoff_24 = t_24 * qrs; { - auto _gst_315 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_o_24; + auto _gst_347 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_o_24; } if (t_24 < clen) { { - auto _gst_316 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_348 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_348) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_317 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_349 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_349) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_25 = 8 * th + 1; @@ -4216,24 +4460,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_25 = dk2_25 * bt_t_25; float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; - float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25) + dgf_25; + float qe_24 = q_t0_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24)) + dgf_25; int goff_25 = t_25 * rs_g; int qoff_25 = t_25 * qrs; { - auto _gst_318 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = dg_o_25; + auto _gst_350 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_350)); + *(reinterpret_cast(_gst_350) + (0)) = dg_o_25; } if (t_25 < clen) { { - auto _gst_319 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_351 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_320 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_352 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_26 = 8 * th + 2; @@ -4259,24 +4508,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_26 = dk2_26 * bt_t_26; float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; - float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26) + dgf_26; + float qe_25 = q_t0_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25)) + dgf_26; int goff_26 = t_26 * rs_g; int qoff_26 = t_26 * qrs; { - auto _gst_321 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast(_gst_321) + (0)) = dg_o_26; + auto _gst_353 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast(_gst_353) + (0)) = dg_o_26; } if (t_26 < clen) { { - auto _gst_322 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_354 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_323 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_355 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_27 = 8 * th + 3; @@ -4302,24 +4556,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_27 = dk2_27 * bt_t_27; float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; - float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27) + dgf_27; + float qe_26 = q_t0_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26)) + dgf_27; int goff_27 = t_27 * rs_g; int qoff_27 = t_27 * qrs; { - auto _gst_324 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast(_gst_324) + (0)) = dg_o_27; + auto _gst_356 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast(_gst_356) + (0)) = dg_o_27; } if (t_27 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_28 = 8 * th + 4; @@ -4345,24 +4604,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_28 = dk2_28 * bt_t_28; float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; - float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28) + dgf_28; + float qe_27 = q_t0_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27)) + dgf_28; int goff_28 = t_28 * rs_g; int qoff_28 = t_28 * qrs; { - auto _gst_327 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast(_gst_327) + (0)) = dg_o_28; + auto _gst_359 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast(_gst_359) + (0)) = dg_o_28; } if (t_28 < clen) { { - auto _gst_328 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_360 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_329 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_361 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_29 = 8 * th + 5; @@ -4388,24 +4652,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_29 = dk2_29 * bt_t_29; float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; - float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29) + dgf_29; + float qe_28 = q_t0_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28)) + dgf_29; int goff_29 = t_29 * rs_g; int qoff_29 = t_29 * qrs; { - auto _gst_330 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast(_gst_330) + (0)) = dg_o_29; + auto _gst_362 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast(_gst_362) + (0)) = dg_o_29; } if (t_29 < clen) { { - auto _gst_331 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_363 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_332 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_364 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_30 = 8 * th + 6; @@ -4431,24 +4700,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_30 = dk2_30 * bt_t_30; float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; - float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30) + dgf_30; + float qe_29 = q_t0_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29)) + dgf_30; int goff_30 = t_30 * rs_g; int qoff_30 = t_30 * qrs; { - auto _gst_333 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast(_gst_333) + (0)) = dg_o_30; + auto _gst_365 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast(_gst_365) + (0)) = dg_o_30; } if (t_30 < clen) { { - auto _gst_334 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_366 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_335 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_335)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_335) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_367 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_367)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_367) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_31 = 8 * th + 7; @@ -4474,24 +4748,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_31 = dk2_31 * bt_t_31; float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; - float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31) + dgf_31; + float qe_30 = q_t0_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30)) + dgf_31; int goff_31 = t_31 * rs_g; int qoff_31 = t_31 * qrs; { - auto _gst_336 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_336)); - *(reinterpret_cast(_gst_336) + (0)) = dg_o_31; + auto _gst_368 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_368)); + *(reinterpret_cast(_gst_368) + (0)) = dg_o_31; } if (t_31 < clen) { { - auto _gst_337 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_337)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_369 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_369)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_369) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_338 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_338)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_338) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_370 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_370)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_370) + (0)) = (__nv_bfloat16)dk_o_31; } } int b4_2 = lane_1 >> 4 & 1; @@ -4499,31 +4778,31 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_371; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_372; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_341; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_373; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_342; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_374; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_343; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_375; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_344; + float _shfl_xor_sync_376 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_376; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_345; - float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_346; - float _shfl_xor_sync_347 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_347; + float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_377; + float _shfl_xor_sync_378 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_378; + float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_379; if ((lane_1 & 3) == 0) { int e_star = lane_1 >> 2 & 7; pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_2; diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu index acde3620584..1e8f10c85cc 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -199,6 +200,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2267,12 +2284,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_4); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_40); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_40, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_40 : __fmaf_rn(dl, _rcp_170, e2_40)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2296,12 +2319,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_5); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_41); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_41, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_41, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_41 : __fmaf_rn(dl_1, _rcp_174, e2_41)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2325,12 +2354,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_6); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_42); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_42, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_42, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_42 : __fmaf_rn(dl_3, _rcp_178, e2_42)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2354,12 +2389,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_7); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_43); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_43, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_43, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_43 : __fmaf_rn(dl_5, _rcp_182, e2_43)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2395,12 +2436,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2420,12 +2467,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2445,12 +2498,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2470,12 +2529,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2523,12 +2588,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_0 = sub_f32x2(_f2_64, _f2_8); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2552,12 +2623,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_1 = sub_f32x2(_f2_64, _f2_9); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2581,12 +2658,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_2 = sub_f32x2(_f2_64, _f2_10); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2610,12 +2693,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_3 = sub_f32x2(_f2_64, _f2_11); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2657,12 +2746,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_4 = sub_f32x2(_f2_70, _f2_12); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2686,12 +2781,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_5 = sub_f32x2(_f2_70, _f2_13); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2715,12 +2816,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_6 = sub_f32x2(_f2_70, _f2_14); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2744,12 +2851,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_7 = sub_f32x2(_f2_70, _f2_15); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2820,37 +2933,37 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_60.x); float e1_12 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_61 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_61.x); float e1_13 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_62 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_62.x); float e1_14 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_63 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_63.x); float e1_15 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2862,29 +2975,29 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2892,29 +3005,29 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2940,885 +3053,801 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int sb8 = 32 + 8 * th; int gb8 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8 * 128); int kb8 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8 * 128); + int c_bp = 144 + 8 * th; + float s_run[1]; + s_run[0] = 0.0f; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j = 7 - k; + float qo[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qo[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + j))); + float ko[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(ko[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th + j))); + float qdg[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qdg[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + j))); + float kdg[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(kdg[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th + j))); + float dkt_0[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(dkt_0[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th + j))); + float btv[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btv[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th + j)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8 + j; + float r_248[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_248[0]; + unsigned int r_249[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_249[0] << 16); + unsigned int r_250[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_250[0] << 16); + float q_t_1 = ((t >= clen) ? 0.0f : q_t); + float bt_t = btv[0]; + float e_off = approx_exp2(g_t - gn); + float e_dg = approx_exp2(g_t - gc_0); + float dq2 = __fmaf_rn(qo[0], e_off, qdg[0] * e_dg); + float dk2 = __fmaf_rn(ko[0], e_off, kdg[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float qe = q_t_1 * e_dg; + float kp = k_t * bt_t; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg) * kdg[0]); + float dqf = pool_f32[ew + 128 * j + d_lane]; + float dkf = pool_f32[ew + (1024 + 128 * j) + d_lane]; + float dgf = pool_f32[ew + (2048 + 128 * j) + d_lane]; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + dkt_0[0]; + float dg_o = __fmaf_rn(q_t_1, dq2, __fmaf_rn(dk2b - dkt_0[0], k_t, cr)) + dgf; + s_run[0] = s_run[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + j)), "f"(s_run[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff = t * qrs; + if (t < clen) { + { + auto _gst_248 = dq_base + qoff; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o; + } + { + auto _gst_249 = dk_base + qoff; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o; + } + } + } + float tot = s_run[0]; float dbp[8]; - float qo[8]; - tmem_ld_x8(&qo[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float ko[8]; - tmem_ld_x8(&ko[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float qdg[8]; - tmem_ld_x8(&qdg[0], taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)); - float kdg[8]; - tmem_ld_x8(&kdg[0], taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th)); - float dkt_0[8]; - tmem_ld_x8(&dkt_0[0], taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th)); - float btv[8]; + tmem_ld_x8(&dbp[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int b4 = lane_1 >> 4 & 1; + int b3 = lane_1 >> 3 & 1; + int b2 = lane_1 >> 2 & 1; + float v4[4]; + float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_250; + float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_251; + float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_252; + float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_253; + float v2[2]; + float send_4 = ((b3 == 0) ? v4[2] : v4[0]); + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_254; + float send_5 = ((b3 == 0) ? v4[3] : v4[1]); + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_255; + float send = ((b2 == 0) ? v2[1] : v2[0]); + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_256; + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_257; + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_258; + asm volatile("tcgen05.fence::before_thread_sync;"); + mbarrier_arrive(p1a_t2_drained_addr); + mbarrier_arrive(epi_empty_addr + (th) * 8); + int pf_row = 32 + 8 * th + (lane_1 & 7) + 16 * (lane_1 >> 3); + long long pf_off = (ext0 + (long long)pf_row) * g_raw_row_stride + (long long)hv * g_raw_head_stride; + if (lane_1 < 16 && pf_row < clen) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(g_raw + (pf_off + (long long)(32 * qd))))); } + if (lane_1 == 16) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(A_log + hv))); } + if (lane_1 == 17) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(dt_bias + (hv * 128 + 32 * qd)))); } + mbarrier_wait(epi_full_addr + (th) * 8, 1); + asm volatile("tcgen05.fence::after_thread_sync;"); + float r_251[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_0 = r_251[0]; + float gc_1 = gn_0 - 21.640425f; + int eb_0 = 98304 + th * -32768; + int ew_0 = eb_0 >> 2; + int qbase_1 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); + int sb8_0 = 48 + 8 * th; + int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); + int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); +#if __CUDA_ARCH__ == 1030 + float dbp_0[8]; + float qo_1[8]; + tmem_ld_x8(&qo_1[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); + float ko_1[8]; + tmem_ld_x8(&ko_1[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); + float qdg_1[8]; + tmem_ld_x8(&qdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); + float kdg_1[8]; + tmem_ld_x8(&kdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); + float dkt_1[8]; + tmem_ld_x8(&dkt_1[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); + float btv_1[8]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[0])), "=r"(*reinterpret_cast(&btv[(0) + 1])), "=r"(*reinterpret_cast(&btv[(0) + 2])), "=r"(*reinterpret_cast(&btv[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th)))); + : "=r"(*reinterpret_cast(&btv_1[0])), "=r"(*reinterpret_cast(&btv_1[(0) + 1])), "=r"(*reinterpret_cast(&btv_1[(0) + 2])), "=r"(*reinterpret_cast(&btv_1[(0) + 3])) + : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[4])), "=r"(*reinterpret_cast(&btv[(4) + 1])), "=r"(*reinterpret_cast(&btv[(4) + 2])), "=r"(*reinterpret_cast(&btv[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th) + 16))); + : "=r"(*reinterpret_cast(&btv_1[4])), "=r"(*reinterpret_cast(&btv_1[(4) + 1])), "=r"(*reinterpret_cast(&btv_1[(4) + 2])), "=r"(*reinterpret_cast(&btv_1[(4) + 3])) + : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run[1]; - s_run[0] = 0.0f; - int t_0 = 32 + 8 * th + 7; - float r_248[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_248[0]; - unsigned int r_249[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_249[0] << 16); - unsigned int r_250[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_250[0] << 16); +#else + int c_bp_0 = 176 + 8 * th; +#endif + float s_run_0[1]; + s_run_0[0] = 0.0f; +#if __CUDA_ARCH__ == 1030 + int t_0 = 48 + 8 * th + 7; + float r_252[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_0 = r_252[0]; + unsigned int r_253[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_0 = __uint_as_float(r_253[0] << 16); + unsigned int r_254[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_0 = __uint_as_float(r_254[0] << 16); float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew + 896 + d_lane]; - float dkf_0 = pool_f32[ew + 1920 + d_lane]; - float dgf_0 = pool_f32[ew + 2944 + d_lane]; - float bt_t_0 = btv[7]; - float e_off_0 = approx_exp2(g_t_0 - gn); - float e_dg_0 = approx_exp2(g_t_0 - gc_0); - float dq2_0 = __fmaf_rn(qo[7], e_off_0, qdg[7] * e_dg_0); - float dk2_0 = __fmaf_rn(ko[7], e_off_0, kdg[7] * e_dg_0); - dbp[7] = dk2_0 * k_t_0; + float bt_t_0 = btv_1[7]; + float e_off_0 = approx_exp2(g_t_0 - gn_0); + float e_dg_0 = approx_exp2(g_t_0 - gc_1); + float dq2_0 = __fmaf_rn(qo_1[7], e_off_0, qdg_1[7] * e_dg_0); + float dk2_0 = __fmaf_rn(ko_1[7], e_off_0, kdg_1[7] * e_dg_0); + dbp_0[7] = dk2_0 * k_t_0; float dk2b_0 = dk2_0 * bt_t_0; + float qe_1 = q_t_1_0 * e_dg_0; + float kp_12 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_12; + float kbe_1 = (float)kb_1 * e_dg_0; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg_1[7], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_12 * e_dg_0) * kdg_1[7]); + float dqf_0 = pool_f32[ew_0 + 896 + d_lane]; + float dkf_0 = pool_f32[ew_0 + 1920 + d_lane]; + float dgf_0 = pool_f32[ew_0 + 2944 + d_lane]; float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + dkt_0[7]; - float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, (dk2b_0 - dkt_0[7]) * k_t_0) + dgf_0; - s_run[0] = s_run[0] + dg_o_0; + float dk_o_0 = dk2b_0 + dkf_0 + dkt_1[7]; + float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_1[7], k_t_0, cr_1)) + dgf_0; + s_run_0[0] = s_run_0[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 7)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_259 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_260 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_0; } } - int t_1 = 32 + 8 * th + 6; - float r_251[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_251[0]; - unsigned int r_252[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((kb8 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_252[0] << 16); - unsigned int r_253[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_253[0] << 16); - float q_t_1_1 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew + 768 + d_lane]; - float dkf_1 = pool_f32[ew + 1792 + d_lane]; - float dgf_1 = pool_f32[ew + 2816 + d_lane]; - float bt_t_1 = btv[6]; - float e_off_1 = approx_exp2(g_t_1 - gn); - float e_dg_1 = approx_exp2(g_t_1 - gc_0); - float dq2_1 = __fmaf_rn(qo[6], e_off_1, qdg[6] * e_dg_1); - float dk2_1 = __fmaf_rn(ko[6], e_off_1, kdg[6] * e_dg_1); - dbp[6] = dk2_1 * k_t_1; + int t_1 = 48 + 8 * th + 6; + float r_255[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_255[0]; + unsigned int r_256[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_256[0] << 16); + unsigned int r_257[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1_1 = __uint_as_float(r_257[0] << 16); + float q_t_1_1_1 = ((t_1 >= clen) ? 0.0f : q_t_1_1); + float bt_t_1 = btv_1[6]; + float e_off_1 = approx_exp2(g_t_1 - gn_0); + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float dq2_1 = __fmaf_rn(qo_1[6], e_off_1, qdg_1[6] * e_dg_1); + float dk2_1 = __fmaf_rn(ko_1[6], e_off_1, kdg_1[6] * e_dg_1); + dbp_0[6] = dk2_1 * k_t_1; float dk2b_1 = dk2_1 * bt_t_1; + float qe_0 = q_t_1_1_1 * e_dg_1; + float kp_12_1 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12_1; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg_1[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12_1 * e_dg_1) * kdg_1[6]); + float dqf_1 = pool_f32[ew_0 + 768 + d_lane]; + float dkf_1 = pool_f32[ew_0 + 1792 + d_lane]; + float dgf_1 = pool_f32[ew_0 + 2816 + d_lane]; float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + dkt_0[6]; - float dg_o_1 = __fmaf_rn(q_t_1_1, dq2_1, (dk2b_1 - dkt_0[6]) * k_t_1) + dgf_1; - s_run[0] = s_run[0] + dg_o_1; + float dk_o_1 = dk2b_1 + dkf_1 + dkt_1[6]; + float dg_o_1 = __fmaf_rn(q_t_1_1_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_1[6], k_t_1, cr_0)) + dgf_1; + s_run_0[0] = s_run_0[0] + dg_o_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 6)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_1 = t_1 * qrs; if (t_1 < clen) { { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_261 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_262 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dk_o_1; } } - int t_2 = 32 + 8 * th + 5; - float r_254[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_254[0]; - unsigned int r_255[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((kb8 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_255[0] << 16); - unsigned int r_256[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_256[0] << 16); + int t_2 = 48 + 8 * th + 5; + float r_258[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_258[0]; + unsigned int r_259[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_259[0] << 16); + unsigned int r_260[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_260[0] << 16); float q_t_1_2 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew + 640 + d_lane]; - float dkf_2 = pool_f32[ew + 1664 + d_lane]; - float dgf_2 = pool_f32[ew + 2688 + d_lane]; - float bt_t_2 = btv[5]; - float e_off_2 = approx_exp2(g_t_2 - gn); - float e_dg_2 = approx_exp2(g_t_2 - gc_0); - float dq2_2 = __fmaf_rn(qo[5], e_off_2, qdg[5] * e_dg_2); - float dk2_2 = __fmaf_rn(ko[5], e_off_2, kdg[5] * e_dg_2); - dbp[5] = dk2_2 * k_t_2; + float bt_t_2 = btv_1[5]; + float e_off_2 = approx_exp2(g_t_2 - gn_0); + float e_dg_2 = approx_exp2(g_t_2 - gc_1); + float dq2_2 = __fmaf_rn(qo_1[5], e_off_2, qdg_1[5] * e_dg_2); + float dk2_2 = __fmaf_rn(ko_1[5], e_off_2, kdg_1[5] * e_dg_2); + dbp_0[5] = dk2_2 * k_t_2; float dk2b_2 = dk2_2 * bt_t_2; + float qe_1_1 = q_t_1_2 * e_dg_2; + float kp_13 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1_1 = (__nv_bfloat16)kp_13; + float kbe_1_1 = (float)kb_1_1 * e_dg_2; + float cr_1_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1_1) & 4294959104u) - qe_1_1, qdg_1[5], (__uint_as_float(__as_u32(kbe_1_1) & 4294959104u) - kp_13 * e_dg_2) * kdg_1[5]); + float dqf_2 = pool_f32[ew_0 + 640 + d_lane]; + float dkf_2 = pool_f32[ew_0 + 1664 + d_lane]; + float dgf_2 = pool_f32[ew_0 + 2688 + d_lane]; float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + dkt_0[5]; - float dg_o_2 = __fmaf_rn(q_t_1_2, dq2_2, (dk2b_2 - dkt_0[5]) * k_t_2) + dgf_2; - s_run[0] = s_run[0] + dg_o_2; + float dk_o_2 = dk2b_2 + dkf_2 + dkt_1[5]; + float dg_o_2 = __fmaf_rn(q_t_1_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_1[5], k_t_2, cr_1_1)) + dgf_2; + s_run_0[0] = s_run_0[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 5)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_263 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_264 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_264)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_264) + (0)) = (__nv_bfloat16)dk_o_2; } } - int t_3 = 32 + 8 * th + 4; - float r_257[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_257[0]; - unsigned int r_258[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((kb8 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_258[0] << 16); - unsigned int r_259[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_259[0] << 16); + int t_3 = 48 + 8 * th + 4; + float r_261[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_3 = r_261[0]; + unsigned int r_262[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_3 = __uint_as_float(r_262[0] << 16); + unsigned int r_263[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_3 = __uint_as_float(r_263[0] << 16); float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew + 512 + d_lane]; - float dkf_3 = pool_f32[ew + 1536 + d_lane]; - float dgf_3 = pool_f32[ew + 2560 + d_lane]; - float bt_t_3 = btv[4]; - float e_off_3 = approx_exp2(g_t_3 - gn); - float e_dg_3 = approx_exp2(g_t_3 - gc_0); - float dq2_3 = __fmaf_rn(qo[4], e_off_3, qdg[4] * e_dg_3); - float dk2_3 = __fmaf_rn(ko[4], e_off_3, kdg[4] * e_dg_3); - dbp[4] = dk2_3 * k_t_3; + float bt_t_3 = btv_1[4]; + float e_off_3 = approx_exp2(g_t_3 - gn_0); + float e_dg_3 = approx_exp2(g_t_3 - gc_1); + float dq2_3 = __fmaf_rn(qo_1[4], e_off_3, qdg_1[4] * e_dg_3); + float dk2_3 = __fmaf_rn(ko_1[4], e_off_3, kdg_1[4] * e_dg_3); + dbp_0[4] = dk2_3 * k_t_3; float dk2b_3 = dk2_3 * bt_t_3; + float qe_2 = q_t_1_3 * e_dg_3; + float kp_14 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg_1[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_3) * kdg_1[4]); + float dqf_3 = pool_f32[ew_0 + 512 + d_lane]; + float dkf_3 = pool_f32[ew_0 + 1536 + d_lane]; + float dgf_3 = pool_f32[ew_0 + 2560 + d_lane]; float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + dkt_0[4]; - float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, (dk2b_3 - dkt_0[4]) * k_t_3) + dgf_3; - s_run[0] = s_run[0] + dg_o_3; + float dk_o_3 = dk2b_3 + dkf_3 + dkt_1[4]; + float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_1[4], k_t_3, cr_2)) + dgf_3; + s_run_0[0] = s_run_0[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 4)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_265 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_265)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_266 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_266)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_266) + (0)) = (__nv_bfloat16)dk_o_3; } } - int t_4 = 32 + 8 * th + 3; - float r_260[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_260[0]; - unsigned int r_261[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((kb8 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_261[0] << 16); - unsigned int r_262[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_262[0] << 16); + int t_4 = 48 + 8 * th + 3; + float r_264[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_4 = r_264[0]; + unsigned int r_265[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_4 = __uint_as_float(r_265[0] << 16); + unsigned int r_266[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_4 = __uint_as_float(r_266[0] << 16); float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew + 384 + d_lane]; - float dkf_4 = pool_f32[ew + 1408 + d_lane]; - float dgf_4 = pool_f32[ew + 2432 + d_lane]; - float bt_t_4 = btv[3]; - float e_off_4 = approx_exp2(g_t_4 - gn); - float e_dg_4 = approx_exp2(g_t_4 - gc_0); - float dq2_4 = __fmaf_rn(qo[3], e_off_4, qdg[3] * e_dg_4); - float dk2_4 = __fmaf_rn(ko[3], e_off_4, kdg[3] * e_dg_4); - dbp[3] = dk2_4 * k_t_4; + float bt_t_4 = btv_1[3]; + float e_off_4 = approx_exp2(g_t_4 - gn_0); + float e_dg_4 = approx_exp2(g_t_4 - gc_1); + float dq2_4 = __fmaf_rn(qo_1[3], e_off_4, qdg_1[3] * e_dg_4); + float dk2_4 = __fmaf_rn(ko_1[3], e_off_4, kdg_1[3] * e_dg_4); + dbp_0[3] = dk2_4 * k_t_4; float dk2b_4 = dk2_4 * bt_t_4; + float qe_3 = q_t_1_4 * e_dg_4; + float kp_15 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg_1[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_4) * kdg_1[3]); + float dqf_4 = pool_f32[ew_0 + 384 + d_lane]; + float dkf_4 = pool_f32[ew_0 + 1408 + d_lane]; + float dgf_4 = pool_f32[ew_0 + 2432 + d_lane]; float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + dkt_0[3]; - float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, (dk2b_4 - dkt_0[3]) * k_t_4) + dgf_4; - s_run[0] = s_run[0] + dg_o_4; + float dk_o_4 = dk2b_4 + dkf_4 + dkt_1[3]; + float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_1[3], k_t_4, cr_3)) + dgf_4; + s_run_0[0] = s_run_0[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 3)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_267 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_267)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_267) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_268 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_268)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dk_o_4; } } - int t_5 = 32 + 8 * th + 2; - float r_263[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_263[0]; - unsigned int r_264[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((kb8 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_264[0] << 16); - unsigned int r_265[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_265[0] << 16); + int t_5 = 48 + 8 * th + 2; + float r_267[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_5 = r_267[0]; + unsigned int r_268[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_5 = __uint_as_float(r_268[0] << 16); + unsigned int r_269[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_5 = __uint_as_float(r_269[0] << 16); float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew + 256 + d_lane]; - float dkf_5 = pool_f32[ew + 1280 + d_lane]; - float dgf_5 = pool_f32[ew + 2304 + d_lane]; - float bt_t_5 = btv[2]; - float e_off_5 = approx_exp2(g_t_5 - gn); - float e_dg_5 = approx_exp2(g_t_5 - gc_0); - float dq2_5 = __fmaf_rn(qo[2], e_off_5, qdg[2] * e_dg_5); - float dk2_5 = __fmaf_rn(ko[2], e_off_5, kdg[2] * e_dg_5); - dbp[2] = dk2_5 * k_t_5; + float bt_t_5 = btv_1[2]; + float e_off_5 = approx_exp2(g_t_5 - gn_0); + float e_dg_5 = approx_exp2(g_t_5 - gc_1); + float dq2_5 = __fmaf_rn(qo_1[2], e_off_5, qdg_1[2] * e_dg_5); + float dk2_5 = __fmaf_rn(ko_1[2], e_off_5, kdg_1[2] * e_dg_5); + dbp_0[2] = dk2_5 * k_t_5; float dk2b_5 = dk2_5 * bt_t_5; + float qe_4 = q_t_1_5 * e_dg_5; + float kp_16 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg_1[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_5) * kdg_1[2]); + float dqf_5 = pool_f32[ew_0 + 256 + d_lane]; + float dkf_5 = pool_f32[ew_0 + 1280 + d_lane]; + float dgf_5 = pool_f32[ew_0 + 2304 + d_lane]; float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + dkt_0[2]; - float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, (dk2b_5 - dkt_0[2]) * k_t_5) + dgf_5; - s_run[0] = s_run[0] + dg_o_5; + float dk_o_5 = dk2b_5 + dkf_5 + dkt_1[2]; + float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_1[2], k_t_5, cr_4)) + dgf_5; + s_run_0[0] = s_run_0[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 2)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_269 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_269)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_269) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_270 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_270)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_270) + (0)) = (__nv_bfloat16)dk_o_5; } } - int t_6 = 32 + 8 * th + 1; - float r_266[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_266[0]; - unsigned int r_267[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_267[0] << 16); - unsigned int r_268[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_268[0] << 16); + int t_6 = 48 + 8 * th + 1; + float r_270[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_6 = r_270[0]; + unsigned int r_271[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_6 = __uint_as_float(r_271[0] << 16); + unsigned int r_272[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_6 = __uint_as_float(r_272[0] << 16); float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew + 128 + d_lane]; - float dkf_6 = pool_f32[ew + 1152 + d_lane]; - float dgf_6 = pool_f32[ew + 2176 + d_lane]; - float bt_t_6 = btv[1]; - float e_off_6 = approx_exp2(g_t_6 - gn); - float e_dg_6 = approx_exp2(g_t_6 - gc_0); - float dq2_6 = __fmaf_rn(qo[1], e_off_6, qdg[1] * e_dg_6); - float dk2_6 = __fmaf_rn(ko[1], e_off_6, kdg[1] * e_dg_6); - dbp[1] = dk2_6 * k_t_6; + float bt_t_6 = btv_1[1]; + float e_off_6 = approx_exp2(g_t_6 - gn_0); + float e_dg_6 = approx_exp2(g_t_6 - gc_1); + float dq2_6 = __fmaf_rn(qo_1[1], e_off_6, qdg_1[1] * e_dg_6); + float dk2_6 = __fmaf_rn(ko_1[1], e_off_6, kdg_1[1] * e_dg_6); + dbp_0[1] = dk2_6 * k_t_6; float dk2b_6 = dk2_6 * bt_t_6; + float qe_5 = q_t_1_6 * e_dg_6; + float kp_17 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg_1[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_6) * kdg_1[1]); + float dqf_6 = pool_f32[ew_0 + 128 + d_lane]; + float dkf_6 = pool_f32[ew_0 + 1152 + d_lane]; + float dgf_6 = pool_f32[ew_0 + 2176 + d_lane]; float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + dkt_0[1]; - float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, (dk2b_6 - dkt_0[1]) * k_t_6) + dgf_6; - s_run[0] = s_run[0] + dg_o_6; + float dk_o_6 = dk2b_6 + dkf_6 + dkt_1[1]; + float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_1[1], k_t_6, cr_5)) + dgf_6; + s_run_0[0] = s_run_0[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 1)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_271 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_271)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_272 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_272)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_272) + (0)) = (__nv_bfloat16)dk_o_6; } } - int t_7 = 32 + 8 * th; - float r_269[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_269[0]; - unsigned int r_270[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((kb8 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_270[0] << 16); - unsigned int r_271[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_271[0] << 16); + int t_7 = 48 + 8 * th; + float r_273[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_7 = r_273[0]; + unsigned int r_274[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_7 = __uint_as_float(r_274[0] << 16); + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_7 = __uint_as_float(r_275[0] << 16); float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew + d_lane]; - float dkf_7 = pool_f32[ew + 1024 + d_lane]; - float dgf_7 = pool_f32[ew + 2048 + d_lane]; - float bt_t_7 = btv[0]; - float e_off_7 = approx_exp2(g_t_7 - gn); - float e_dg_7 = approx_exp2(g_t_7 - gc_0); - float dq2_7 = __fmaf_rn(qo[0], e_off_7, qdg[0] * e_dg_7); - float dk2_7 = __fmaf_rn(ko[0], e_off_7, kdg[0] * e_dg_7); - dbp[0] = dk2_7 * k_t_7; + float bt_t_7 = btv_1[0]; + float e_off_7 = approx_exp2(g_t_7 - gn_0); + float e_dg_7 = approx_exp2(g_t_7 - gc_1); + float dq2_7 = __fmaf_rn(qo_1[0], e_off_7, qdg_1[0] * e_dg_7); + float dk2_7 = __fmaf_rn(ko_1[0], e_off_7, kdg_1[0] * e_dg_7); + dbp_0[0] = dk2_7 * k_t_7; float dk2b_7 = dk2_7 * bt_t_7; + float qe_6 = q_t_1_7 * e_dg_7; + float kp_18 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg_1[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_7) * kdg_1[0]); + float dqf_7 = pool_f32[ew_0 + d_lane]; + float dkf_7 = pool_f32[ew_0 + 1024 + d_lane]; + float dgf_7 = pool_f32[ew_0 + 2048 + d_lane]; float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + dkt_0[0]; - float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, (dk2b_7 - dkt_0[0]) * k_t_7) + dgf_7; - s_run[0] = s_run[0] + dg_o_7; + float dk_o_7 = dk2b_7 + dkf_7 + dkt_1[0]; + float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_1[0], k_t_7, cr_6)) + dgf_7; + s_run_0[0] = s_run_0[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_273 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_274 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_7; +#else + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_1 = 7 - k_1; + float qo_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qo_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th + j_1))); + float ko_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(ko_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th + j_1))); + float qdg_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qdg_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + j_1))); + float kdg_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(kdg_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th + j_1))); + float dkt_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(dkt_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th + j_1))); + float btv_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btv_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_0 + j_1; + float r_252[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_252[0]; + unsigned int r_253[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_253[0] << 16); + unsigned int r_254[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_254[0] << 16); + float q_t_1_1 = ((t_1 >= clen) ? 0.0f : q_t_2); + float bt_t_1 = btv_1[0]; + float e_off_1 = approx_exp2(g_t_1 - gn_0); + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float dq2_1 = __fmaf_rn(qo_1[0], e_off_1, qdg_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(ko_1[0], e_off_1, kdg_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float qe_1 = q_t_1_1 * e_dg_1; + float kp_12 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_12; + float kbe_1 = (float)kb_1 * e_dg_1; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg_1[0], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_12 * e_dg_1) * kdg_1[0]); + float dqf_1 = pool_f32[ew_0 + 128 * j_1 + d_lane]; + float dkf_1 = pool_f32[ew_0 + (1024 + 128 * j_1) + d_lane]; + float dgf_1 = pool_f32[ew_0 + (2048 + 128 * j_1) + d_lane]; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + dkt_1[0]; + float dg_o_1 = __fmaf_rn(q_t_1_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_1[0], k_t_1, cr_1)) + dgf_1; + s_run_0[0] = s_run_0[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + j_1)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_1)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff_1 = t_1 * qrs; + if (t_1 < clen) { + { + auto _gst_259 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_1; + } + { + auto _gst_260 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_1; + } +#endif } } - float tot = s_run[0]; - int b4 = lane_1 >> 4 & 1; - int b3 = lane_1 >> 3 & 1; - int b2 = lane_1 >> 2 & 1; - float v4[4]; - float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; - float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; - float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; - float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; - float v2[2]; - float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; - float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; - float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float tot_0 = s_run_0[0]; +#if __CUDA_ARCH__ == 1030 +#else + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); +#endif + int b4_0 = lane_1 >> 4 & 1; + int b3_0 = lane_1 >> 3 & 1; + int b2_0 = lane_1 >> 2 & 1; + float v4_0[4]; + float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_275; +#else + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_261; +#endif + float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_276; +#else + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_262; +#endif + float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_277; +#else + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_263; +#endif + float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_278; +#else + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_264; +#endif + float v2_16[2]; + float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_279; +#else + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_265; +#endif + float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_280; +#else + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_266; +#endif + float send_12 = ((b2_0 == 0) ? v2_16[1] : v2_16[0]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_281; + float _shfl_xor_sync_282 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_282; + float _shfl_xor_sync_283 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_283; +#else + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_267; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_268; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_269; +#endif asm volatile("tcgen05.fence::before_thread_sync;"); - mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); - int pf_row = 32 + 8 * th + (lane_1 & 7) + 16 * (lane_1 >> 3); - long long pf_off = (ext0 + (long long)pf_row) * g_raw_row_stride + (long long)hv * g_raw_head_stride; - if (lane_1 < 16 && pf_row < clen) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(g_raw + (pf_off + (long long)(32 * qd))))); } - if (lane_1 == 16) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(A_log + hv))); } - if (lane_1 == 17) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(dt_bias + (hv * 128 + 32 * qd)))); } - mbarrier_wait(epi_full_addr + (th) * 8, 1); - asm volatile("tcgen05.fence::after_thread_sync;"); - float r_272[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_0 = r_272[0]; - float gc_1 = gn_0 - 21.640425f; - int eb_0 = 98304 + th * -32768; - int ew_0 = eb_0 >> 2; - int qbase_1 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); - int sb8_0 = 48 + 8 * th; - int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); - int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); - float dbp_0[8]; - float qo_0[8]; - tmem_ld_x8(&qo_0[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float ko_0[8]; - tmem_ld_x8(&ko_0[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float qdg_0[8]; - tmem_ld_x8(&qdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float kdg_0[8]; - tmem_ld_x8(&kdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float dkt_1[8]; - tmem_ld_x8(&dkt_1[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_0[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_0[0])), "=r"(*reinterpret_cast(&btv_0[(0) + 1])), "=r"(*reinterpret_cast(&btv_0[(0) + 2])), "=r"(*reinterpret_cast(&btv_0[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_0[4])), "=r"(*reinterpret_cast(&btv_0[(4) + 1])), "=r"(*reinterpret_cast(&btv_0[(4) + 2])), "=r"(*reinterpret_cast(&btv_0[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_0[1]; - s_run_0[0] = 0.0f; - int t_8 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_0 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_0 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_0 + 2944 + d_lane]; - float bt_t_8 = btv_0[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_0); - float e_dg_8 = approx_exp2(g_t_8 - gc_1); - float dq2_8 = __fmaf_rn(qo_0[7], e_off_8, qdg_0[7] * e_dg_8); - float dk2_8 = __fmaf_rn(ko_0[7], e_off_8, kdg_0[7] * e_dg_8); - dbp_0[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + dkt_1[7]; - float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, (dk2b_8 - dkt_1[7]) * k_t_8) + dgf_8; - s_run_0[0] = s_run_0[0] + dg_o_8; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_8 = t_8 * qrs; - if (t_8 < clen) { - { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; - } - { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; - } - } - int t_9 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_0 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_0 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_0 + 2816 + d_lane]; - float bt_t_9 = btv_0[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_0); - float e_dg_9 = approx_exp2(g_t_9 - gc_1); - float dq2_9 = __fmaf_rn(qo_0[6], e_off_9, qdg_0[6] * e_dg_9); - float dk2_9 = __fmaf_rn(ko_0[6], e_off_9, kdg_0[6] * e_dg_9); - dbp_0[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + dkt_1[6]; - float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, (dk2b_9 - dkt_1[6]) * k_t_9) + dgf_9; - s_run_0[0] = s_run_0[0] + dg_o_9; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_9 = t_9 * qrs; - if (t_9 < clen) { - { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; - } - { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; - } - } - int t_10 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_0 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_0 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_0 + 2688 + d_lane]; - float bt_t_10 = btv_0[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_0); - float e_dg_10 = approx_exp2(g_t_10 - gc_1); - float dq2_10 = __fmaf_rn(qo_0[5], e_off_10, qdg_0[5] * e_dg_10); - float dk2_10 = __fmaf_rn(ko_0[5], e_off_10, kdg_0[5] * e_dg_10); - dbp_0[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + dkt_1[5]; - float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, (dk2b_10 - dkt_1[5]) * k_t_10) + dgf_10; - s_run_0[0] = s_run_0[0] + dg_o_10; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_10 = t_10 * qrs; - if (t_10 < clen) { - { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; - } - { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; - } - } - int t_11 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_0 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_0 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_0 + 2560 + d_lane]; - float bt_t_11 = btv_0[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_0); - float e_dg_11 = approx_exp2(g_t_11 - gc_1); - float dq2_11 = __fmaf_rn(qo_0[4], e_off_11, qdg_0[4] * e_dg_11); - float dk2_11 = __fmaf_rn(ko_0[4], e_off_11, kdg_0[4] * e_dg_11); - dbp_0[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + dkt_1[4]; - float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, (dk2b_11 - dkt_1[4]) * k_t_11) + dgf_11; - s_run_0[0] = s_run_0[0] + dg_o_11; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_11 = t_11 * qrs; - if (t_11 < clen) { - { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; - } - { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; - } - } - int t_12 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_0 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_0 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_0 + 2432 + d_lane]; - float bt_t_12 = btv_0[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_0); - float e_dg_12 = approx_exp2(g_t_12 - gc_1); - float dq2_12 = __fmaf_rn(qo_0[3], e_off_12, qdg_0[3] * e_dg_12); - float dk2_12 = __fmaf_rn(ko_0[3], e_off_12, kdg_0[3] * e_dg_12); - dbp_0[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + dkt_1[3]; - float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, (dk2b_12 - dkt_1[3]) * k_t_12) + dgf_12; - s_run_0[0] = s_run_0[0] + dg_o_12; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_12 = t_12 * qrs; - if (t_12 < clen) { - { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; - } - { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; - } - } - int t_13 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_0 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_0 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_0 + 2304 + d_lane]; - float bt_t_13 = btv_0[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_0); - float e_dg_13 = approx_exp2(g_t_13 - gc_1); - float dq2_13 = __fmaf_rn(qo_0[2], e_off_13, qdg_0[2] * e_dg_13); - float dk2_13 = __fmaf_rn(ko_0[2], e_off_13, kdg_0[2] * e_dg_13); - dbp_0[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + dkt_1[2]; - float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, (dk2b_13 - dkt_1[2]) * k_t_13) + dgf_13; - s_run_0[0] = s_run_0[0] + dg_o_13; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_13 = t_13 * qrs; - if (t_13 < clen) { - { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; - } - { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; - } - } - int t_14 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_0 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_0 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_0 + 2176 + d_lane]; - float bt_t_14 = btv_0[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_0); - float e_dg_14 = approx_exp2(g_t_14 - gc_1); - float dq2_14 = __fmaf_rn(qo_0[1], e_off_14, qdg_0[1] * e_dg_14); - float dk2_14 = __fmaf_rn(ko_0[1], e_off_14, kdg_0[1] * e_dg_14); - dbp_0[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + dkt_1[1]; - float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, (dk2b_14 - dkt_1[1]) * k_t_14) + dgf_14; - s_run_0[0] = s_run_0[0] + dg_o_14; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_14 = t_14 * qrs; - if (t_14 < clen) { - { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; - } - { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; - } - } - int t_15 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_0 + d_lane]; - float dkf_15 = pool_f32[ew_0 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_0 + 2048 + d_lane]; - float bt_t_15 = btv_0[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_0); - float e_dg_15 = approx_exp2(g_t_15 - gc_1); - float dq2_15 = __fmaf_rn(qo_0[0], e_off_15, qdg_0[0] * e_dg_15); - float dk2_15 = __fmaf_rn(ko_0[0], e_off_15, kdg_0[0] * e_dg_15); - dbp_0[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + dkt_1[0]; - float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, (dk2b_15 - dkt_1[0]) * k_t_15) + dgf_15; - s_run_0[0] = s_run_0[0] + dg_o_15; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_15 = t_15 * qrs; - if (t_15 < clen) { - { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; - } - { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; - } - } - float tot_0 = s_run_0[0]; - int b4_0 = lane_1 >> 4 & 1; - int b3_0 = lane_1 >> 3 & 1; - int b2_0 = lane_1 >> 2 & 1; - float v4_0[4]; - float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; - float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; - float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; - float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; - float v2_16[2]; - float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; - float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; - float send_12 = ((b2_0 == 0) ? v2_16[1] : v2_16[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; - asm volatile("tcgen05.fence::before_thread_sync;"); - mbarrier_arrive(epi_empty_addr + (th) * 8); - float gacc[8]; - float rg_rate = __nv_expf(A_log[hv]); - float rg_clr; - asm("mul.rn.ftz.f32 %0, %1, %2;" : "=f"(rg_clr) : "f"(lower_bound), "f"(rg_rate)); - gacc[0] = 0.0f; - gacc[1] = 0.0f; - gacc[2] = 0.0f; - gacc[3] = 0.0f; - gacc[4] = rg_rate; - gacc[5] = rg_clr; - gacc[6] = dt_bias[hv * 128 + d_lane]; - gacc[7] = 0.0f; - float raw_t3[8]; - long long rb = ext0 * g_raw_row_stride + (long long)hv * g_raw_head_stride + (long long)d_lane; - int t_c_0 = ((48 + 8 * th < clen) ? 48 + 8 * th : clen - 1); - raw_t3[0] = (float)g_raw[rb + (long long)t_c_0 * g_raw_row_stride]; - int t_c_1 = ((48 + 8 * th + 1 < clen) ? 48 + 8 * th + 1 : clen - 1); - raw_t3[1] = (float)g_raw[rb + (long long)t_c_1 * g_raw_row_stride]; - int t_c_2 = ((48 + 8 * th + 2 < clen) ? 48 + 8 * th + 2 : clen - 1); - raw_t3[2] = (float)g_raw[rb + (long long)t_c_2 * g_raw_row_stride]; - int t_c_3 = ((48 + 8 * th + 3 < clen) ? 48 + 8 * th + 3 : clen - 1); - raw_t3[3] = (float)g_raw[rb + (long long)t_c_3 * g_raw_row_stride]; - int t_c_4 = ((48 + 8 * th + 4 < clen) ? 48 + 8 * th + 4 : clen - 1); - raw_t3[4] = (float)g_raw[rb + (long long)t_c_4 * g_raw_row_stride]; - int t_c_5 = ((48 + 8 * th + 5 < clen) ? 48 + 8 * th + 5 : clen - 1); - raw_t3[5] = (float)g_raw[rb + (long long)t_c_5 * g_raw_row_stride]; - int t_c_6 = ((48 + 8 * th + 6 < clen) ? 48 + 8 * th + 6 : clen - 1); - raw_t3[6] = (float)g_raw[rb + (long long)t_c_6 * g_raw_row_stride]; - int t_c_7 = ((48 + 8 * th + 7 < clen) ? 48 + 8 * th + 7 : clen - 1); - raw_t3[7] = (float)g_raw[rb + (long long)t_c_7 * g_raw_row_stride]; - float raw_t2[8]; - long long rb_0 = ext0 * g_raw_row_stride + (long long)hv * g_raw_head_stride + (long long)d_lane; - int t_c_8 = ((32 + 8 * th < clen) ? 32 + 8 * th : clen - 1); - raw_t2[0] = (float)g_raw[rb_0 + (long long)t_c_8 * g_raw_row_stride]; - int t_c_9 = ((32 + 8 * th + 1 < clen) ? 32 + 8 * th + 1 : clen - 1); - raw_t2[1] = (float)g_raw[rb_0 + (long long)t_c_9 * g_raw_row_stride]; - int t_c_10 = ((32 + 8 * th + 2 < clen) ? 32 + 8 * th + 2 : clen - 1); - raw_t2[2] = (float)g_raw[rb_0 + (long long)t_c_10 * g_raw_row_stride]; - int t_c_11 = ((32 + 8 * th + 3 < clen) ? 32 + 8 * th + 3 : clen - 1); - raw_t2[3] = (float)g_raw[rb_0 + (long long)t_c_11 * g_raw_row_stride]; - int t_c_12 = ((32 + 8 * th + 4 < clen) ? 32 + 8 * th + 4 : clen - 1); - raw_t2[4] = (float)g_raw[rb_0 + (long long)t_c_12 * g_raw_row_stride]; - int t_c_13 = ((32 + 8 * th + 5 < clen) ? 32 + 8 * th + 5 : clen - 1); - raw_t2[5] = (float)g_raw[rb_0 + (long long)t_c_13 * g_raw_row_stride]; - int t_c_14 = ((32 + 8 * th + 6 < clen) ? 32 + 8 * th + 6 : clen - 1); - raw_t2[6] = (float)g_raw[rb_0 + (long long)t_c_14 * g_raw_row_stride]; - int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); - raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; - { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); - } - asm volatile("tcgen05.fence::before_thread_sync;"); - if (qd == 0) { - asm volatile("barrier.sync 2, 64;" ::: "memory"); - } - if (qd == 1) { - asm volatile("barrier.sync 3, 64;" ::: "memory"); - } - if (qd == 2) { - asm volatile("barrier.sync 4, 64;" ::: "memory"); - } - if (qd == 3) { - asm volatile("barrier.sync 5, 64;" ::: "memory"); - } + float gacc[8]; + float rg_rate = __nv_expf(A_log[hv]); + float rg_clr; + asm("mul.rn.ftz.f32 %0, %1, %2;" : "=f"(rg_clr) : "f"(lower_bound), "f"(rg_rate)); + gacc[0] = 0.0f; + gacc[1] = 0.0f; + gacc[2] = 0.0f; + gacc[3] = 0.0f; + gacc[4] = rg_rate; + gacc[5] = rg_clr; + gacc[6] = dt_bias[hv * 128 + d_lane]; + gacc[7] = 0.0f; + float raw_t3[8]; + long long rb = ext0 * g_raw_row_stride + (long long)hv * g_raw_head_stride + (long long)d_lane; + int t_c_0 = ((48 + 8 * th < clen) ? 48 + 8 * th : clen - 1); + raw_t3[0] = (float)g_raw[rb + (long long)t_c_0 * g_raw_row_stride]; + int t_c_1 = ((48 + 8 * th + 1 < clen) ? 48 + 8 * th + 1 : clen - 1); + raw_t3[1] = (float)g_raw[rb + (long long)t_c_1 * g_raw_row_stride]; + int t_c_2 = ((48 + 8 * th + 2 < clen) ? 48 + 8 * th + 2 : clen - 1); + raw_t3[2] = (float)g_raw[rb + (long long)t_c_2 * g_raw_row_stride]; + int t_c_3 = ((48 + 8 * th + 3 < clen) ? 48 + 8 * th + 3 : clen - 1); + raw_t3[3] = (float)g_raw[rb + (long long)t_c_3 * g_raw_row_stride]; + int t_c_4 = ((48 + 8 * th + 4 < clen) ? 48 + 8 * th + 4 : clen - 1); + raw_t3[4] = (float)g_raw[rb + (long long)t_c_4 * g_raw_row_stride]; + int t_c_5 = ((48 + 8 * th + 5 < clen) ? 48 + 8 * th + 5 : clen - 1); + raw_t3[5] = (float)g_raw[rb + (long long)t_c_5 * g_raw_row_stride]; + int t_c_6 = ((48 + 8 * th + 6 < clen) ? 48 + 8 * th + 6 : clen - 1); + raw_t3[6] = (float)g_raw[rb + (long long)t_c_6 * g_raw_row_stride]; + int t_c_7 = ((48 + 8 * th + 7 < clen) ? 48 + 8 * th + 7 : clen - 1); + raw_t3[7] = (float)g_raw[rb + (long long)t_c_7 * g_raw_row_stride]; + float raw_t2[8]; + long long rb_0 = ext0 * g_raw_row_stride + (long long)hv * g_raw_head_stride + (long long)d_lane; + int t_c_8 = ((32 + 8 * th < clen) ? 32 + 8 * th : clen - 1); + raw_t2[0] = (float)g_raw[rb_0 + (long long)t_c_8 * g_raw_row_stride]; + int t_c_9 = ((32 + 8 * th + 1 < clen) ? 32 + 8 * th + 1 : clen - 1); + raw_t2[1] = (float)g_raw[rb_0 + (long long)t_c_9 * g_raw_row_stride]; + int t_c_10 = ((32 + 8 * th + 2 < clen) ? 32 + 8 * th + 2 : clen - 1); + raw_t2[2] = (float)g_raw[rb_0 + (long long)t_c_10 * g_raw_row_stride]; + int t_c_11 = ((32 + 8 * th + 3 < clen) ? 32 + 8 * th + 3 : clen - 1); + raw_t2[3] = (float)g_raw[rb_0 + (long long)t_c_11 * g_raw_row_stride]; + int t_c_12 = ((32 + 8 * th + 4 < clen) ? 32 + 8 * th + 4 : clen - 1); + raw_t2[4] = (float)g_raw[rb_0 + (long long)t_c_12 * g_raw_row_stride]; + int t_c_13 = ((32 + 8 * th + 5 < clen) ? 32 + 8 * th + 5 : clen - 1); + raw_t2[5] = (float)g_raw[rb_0 + (long long)t_c_13 * g_raw_row_stride]; + int t_c_14 = ((32 + 8 * th + 6 < clen) ? 32 + 8 * th + 6 : clen - 1); + raw_t2[6] = (float)g_raw[rb_0 + (long long)t_c_14 * g_raw_row_stride]; + int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); + raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; + { +#if __CUDA_ARCH__ == 1030 + uint32_t _addr_284 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_284), "f"(tot_0) : "memory"); +#else + uint32_t _addr_270 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_270), "f"(tot_0) : "memory"); +#endif + } + asm volatile("tcgen05.fence::before_thread_sync;"); + if (qd == 0) { + asm volatile("barrier.sync 2, 64;" ::: "memory"); + } + if (qd == 1) { + asm volatile("barrier.sync 3, 64;" ::: "memory"); + } + if (qd == 2) { + asm volatile("barrier.sync 4, 64;" ::: "memory"); + } + if (qd == 3) { + asm volatile("barrier.sync 5, 64;" ::: "memory"); + } asm volatile("tcgen05.fence::after_thread_sync;"); float t_other[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); @@ -3836,132 +3865,280 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_267 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_285 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_271 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&rp_u)); - float _abs_270; - asm("abs.f32 %0, %1;" : "=f"(_abs_270) : "f"(rp_z.x)); - float _neg_269; - asm("neg.f32 %0, %1;" : "=f"(_neg_269) : "f"(_abs_270)); - float _exp_268 = __nv_expf(_neg_269); - float _abs_273; - asm("abs.f32 %0, %1;" : "=f"(_abs_273) : "f"(rp_z.y)); - float _neg_272; - asm("neg.f32 %0, %1;" : "=f"(_neg_272) : "f"(_abs_273)); - float _exp_271 = __nv_expf(_neg_272); - float2 rp_e = make_float2(_exp_268, _exp_271); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_285), "l"(*(const unsigned long long*)&rp_u)); + float _abs_288; + asm("abs.f32 %0, %1;" : "=f"(_abs_288) : "f"(rp_z.x)); + float _neg_287; + asm("neg.f32 %0, %1;" : "=f"(_neg_287) : "f"(_abs_288)); + float _exp_286 = __nv_expf(_neg_287); + float _abs_291; + asm("abs.f32 %0, %1;" : "=f"(_abs_291) : "f"(rp_z.y)); + float _neg_290; + asm("neg.f32 %0, %1;" : "=f"(_neg_290) : "f"(_abs_291)); + float _exp_289 = __nv_expf(_neg_290); + float2 rp_e = make_float2(_exp_286, _exp_289); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_271), "l"(*(const unsigned long long*)&rp_u)); + float _abs_274; + asm("abs.f32 %0, %1;" : "=f"(_abs_274) : "f"(rp_z.x)); + float _neg_273; + asm("neg.f32 %0, %1;" : "=f"(_neg_273) : "f"(_abs_274)); + float _exp_272 = __nv_expf(_neg_273); + float _abs_277; + asm("abs.f32 %0, %1;" : "=f"(_abs_277) : "f"(rp_z.y)); + float _neg_276; + asm("neg.f32 %0, %1;" : "=f"(_neg_276) : "f"(_abs_277)); + float _exp_275 = __nv_expf(_neg_276); + float2 rp_e = make_float2(_exp_272, _exp_275); +#endif float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_274 = approx_rcp(rp_den.x); - float _rcp_275 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_276 = make_float2(_rcp_274, _rcp_275); +#if __CUDA_ARCH__ == 1030 + float _rcp_292 = approx_rcp(rp_den.x); + float _rcp_293 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_294 = make_float2(_rcp_292, _rcp_293); +#else + float _rcp_278 = approx_rcp(rp_den.x); + float _rcp_279 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_280 = make_float2(_rcp_278, _rcp_279); +#endif float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_276)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_294)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_280)); +#endif float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_277 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_295 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_281 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_295)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_281)); +#endif u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_278 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_296 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_282 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_278), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_281; - asm("abs.f32 %0, %1;" : "=f"(_abs_281) : "f"(rp_z_0.x)); - float _neg_280; - asm("neg.f32 %0, %1;" : "=f"(_neg_280) : "f"(_abs_281)); - float _exp_279 = __nv_expf(_neg_280); - float _abs_284; - asm("abs.f32 %0, %1;" : "=f"(_abs_284) : "f"(rp_z_0.y)); - float _neg_283; - asm("neg.f32 %0, %1;" : "=f"(_neg_283) : "f"(_abs_284)); - float _exp_282 = __nv_expf(_neg_283); - float2 rp_e_0 = make_float2(_exp_279, _exp_282); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_296), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_299; + asm("abs.f32 %0, %1;" : "=f"(_abs_299) : "f"(rp_z_0.x)); + float _neg_298; + asm("neg.f32 %0, %1;" : "=f"(_neg_298) : "f"(_abs_299)); + float _exp_297 = __nv_expf(_neg_298); + float _abs_302; + asm("abs.f32 %0, %1;" : "=f"(_abs_302) : "f"(rp_z_0.y)); + float _neg_301; + asm("neg.f32 %0, %1;" : "=f"(_neg_301) : "f"(_abs_302)); + float _exp_300 = __nv_expf(_neg_301); + float2 rp_e_0 = make_float2(_exp_297, _exp_300); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_282), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_285; + asm("abs.f32 %0, %1;" : "=f"(_abs_285) : "f"(rp_z_0.x)); + float _neg_284; + asm("neg.f32 %0, %1;" : "=f"(_neg_284) : "f"(_abs_285)); + float _exp_283 = __nv_expf(_neg_284); + float _abs_288; + asm("abs.f32 %0, %1;" : "=f"(_abs_288) : "f"(rp_z_0.y)); + float _neg_287; + asm("neg.f32 %0, %1;" : "=f"(_neg_287) : "f"(_abs_288)); + float _exp_286 = __nv_expf(_neg_287); + float2 rp_e_0 = make_float2(_exp_283, _exp_286); +#endif float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_285 = approx_rcp(rp_den_0.x); - float _rcp_286 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_287 = make_float2(_rcp_285, _rcp_286); +#if __CUDA_ARCH__ == 1030 + float _rcp_303 = approx_rcp(rp_den_0.x); + float _rcp_304 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_305 = make_float2(_rcp_303, _rcp_304); +#else + float _rcp_289 = approx_rcp(rp_den_0.x); + float _rcp_290 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_291 = make_float2(_rcp_289, _rcp_290); +#endif float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_287)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_305)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_291)); +#endif float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_288 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_306 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_292 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_288)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_306)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_292)); +#endif u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_289 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_307 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_293 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_289), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_292; - asm("abs.f32 %0, %1;" : "=f"(_abs_292) : "f"(rp_z_1.x)); - float _neg_291; - asm("neg.f32 %0, %1;" : "=f"(_neg_291) : "f"(_abs_292)); - float _exp_290 = __nv_expf(_neg_291); - float _abs_295; - asm("abs.f32 %0, %1;" : "=f"(_abs_295) : "f"(rp_z_1.y)); - float _neg_294; - asm("neg.f32 %0, %1;" : "=f"(_neg_294) : "f"(_abs_295)); - float _exp_293 = __nv_expf(_neg_294); - float2 rp_e_1 = make_float2(_exp_290, _exp_293); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_307), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_310; + asm("abs.f32 %0, %1;" : "=f"(_abs_310) : "f"(rp_z_1.x)); + float _neg_309; + asm("neg.f32 %0, %1;" : "=f"(_neg_309) : "f"(_abs_310)); + float _exp_308 = __nv_expf(_neg_309); + float _abs_313; + asm("abs.f32 %0, %1;" : "=f"(_abs_313) : "f"(rp_z_1.y)); + float _neg_312; + asm("neg.f32 %0, %1;" : "=f"(_neg_312) : "f"(_abs_313)); + float _exp_311 = __nv_expf(_neg_312); + float2 rp_e_1 = make_float2(_exp_308, _exp_311); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_293), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_296; + asm("abs.f32 %0, %1;" : "=f"(_abs_296) : "f"(rp_z_1.x)); + float _neg_295; + asm("neg.f32 %0, %1;" : "=f"(_neg_295) : "f"(_abs_296)); + float _exp_294 = __nv_expf(_neg_295); + float _abs_299; + asm("abs.f32 %0, %1;" : "=f"(_abs_299) : "f"(rp_z_1.y)); + float _neg_298; + asm("neg.f32 %0, %1;" : "=f"(_neg_298) : "f"(_abs_299)); + float _exp_297 = __nv_expf(_neg_298); + float2 rp_e_1 = make_float2(_exp_294, _exp_297); +#endif float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_296 = approx_rcp(rp_den_1.x); - float _rcp_297 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_298 = make_float2(_rcp_296, _rcp_297); +#if __CUDA_ARCH__ == 1030 + float _rcp_314 = approx_rcp(rp_den_1.x); + float _rcp_315 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_316 = make_float2(_rcp_314, _rcp_315); +#else + float _rcp_300 = approx_rcp(rp_den_1.x); + float _rcp_301 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_302 = make_float2(_rcp_300, _rcp_301); +#endif float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_298)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_316)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_302)); +#endif float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_299 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_317 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_303 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_299)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_317)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_303)); +#endif u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_300 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_318 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_304 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_303; - asm("abs.f32 %0, %1;" : "=f"(_abs_303) : "f"(rp_z_2.x)); - float _neg_302; - asm("neg.f32 %0, %1;" : "=f"(_neg_302) : "f"(_abs_303)); - float _exp_301 = __nv_expf(_neg_302); - float _abs_306; - asm("abs.f32 %0, %1;" : "=f"(_abs_306) : "f"(rp_z_2.y)); - float _neg_305; - asm("neg.f32 %0, %1;" : "=f"(_neg_305) : "f"(_abs_306)); - float _exp_304 = __nv_expf(_neg_305); - float2 rp_e_2 = make_float2(_exp_301, _exp_304); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_318), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_321; + asm("abs.f32 %0, %1;" : "=f"(_abs_321) : "f"(rp_z_2.x)); + float _neg_320; + asm("neg.f32 %0, %1;" : "=f"(_neg_320) : "f"(_abs_321)); + float _exp_319 = __nv_expf(_neg_320); + float _abs_324; + asm("abs.f32 %0, %1;" : "=f"(_abs_324) : "f"(rp_z_2.y)); + float _neg_323; + asm("neg.f32 %0, %1;" : "=f"(_neg_323) : "f"(_abs_324)); + float _exp_322 = __nv_expf(_neg_323); + float2 rp_e_2 = make_float2(_exp_319, _exp_322); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_304), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_307; + asm("abs.f32 %0, %1;" : "=f"(_abs_307) : "f"(rp_z_2.x)); + float _neg_306; + asm("neg.f32 %0, %1;" : "=f"(_neg_306) : "f"(_abs_307)); + float _exp_305 = __nv_expf(_neg_306); + float _abs_310; + asm("abs.f32 %0, %1;" : "=f"(_abs_310) : "f"(rp_z_2.y)); + float _neg_309; + asm("neg.f32 %0, %1;" : "=f"(_neg_309) : "f"(_abs_310)); + float _exp_308 = __nv_expf(_neg_309); + float2 rp_e_2 = make_float2(_exp_305, _exp_308); +#endif float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_307 = approx_rcp(rp_den_2.x); - float _rcp_308 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_309 = make_float2(_rcp_307, _rcp_308); +#if __CUDA_ARCH__ == 1030 + float _rcp_325 = approx_rcp(rp_den_2.x); + float _rcp_326 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_327 = make_float2(_rcp_325, _rcp_326); +#else + float _rcp_311 = approx_rcp(rp_den_2.x); + float _rcp_312 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_313 = make_float2(_rcp_311, _rcp_312); +#endif float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_327)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_313)); +#endif float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_310 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_328 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_314 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_310)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_328)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_314)); +#endif u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -3976,54 +4153,102 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_311; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_311) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_311; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_329; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_329) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_329; +#else + uint32_t _prmt_315; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_315; +#endif int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_312 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = pq_0[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_330 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast(_gst_330) + (0)) = pq_0[0]; +#else + auto _gst_316 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = pq_0[0]; +#endif } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_313; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_313) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_313; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_331; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_331) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_331; +#else + uint32_t _prmt_317; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_317; +#endif int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_314 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = pq_1[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_332 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_332)); + *(reinterpret_cast(_gst_332) + (0)) = pq_1[0]; +#else + auto _gst_318 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = pq_1[0]; +#endif } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_315; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_315; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_333; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_333) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_333; +#else + uint32_t _prmt_319; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_319) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_319; +#endif int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_316 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = pq_2[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_334 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_334)); + *(reinterpret_cast(_gst_334) + (0)) = pq_2[0]; +#else + auto _gst_320 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = pq_2[0]; +#endif } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_317; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_317; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_335; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_335) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_335; +#else + uint32_t _prmt_321; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_321) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_321; +#endif int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_318 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = pq_3[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_336 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_336)); + *(reinterpret_cast(_gst_336) + (0)) = pq_3[0]; +#else + auto _gst_322 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast(_gst_322) + (0)) = pq_3[0]; +#endif } } int t_a_0 = 48 + 8 * th; @@ -4088,47 +4313,99 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_319; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_319) : "f"(s_a), "f"(nm_t)); +#if __CUDA_ARCH__ == 1030 + float _sub_337; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_337) : "f"(s_a), "f"(nm_t)); +#else + float _sub_323; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_323) : "f"(s_a), "f"(nm_t)); +#endif float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_319), "f"(pa)); - float _sub_320; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_320) : "f"(pa), "f"(nm_t)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_337), "f"(pa)); + float _sub_338; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_338) : "f"(pa), "f"(nm_t)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_323), "f"(pa)); + float _sub_324; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_324) : "f"(pa), "f"(nm_t)); +#endif float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_320), "f"(s_a)); - float _abs_322; - asm("abs.f32 %0, %1;" : "=f"(_abs_322) : "f"(s_a)); - float _abs_323; - asm("abs.f32 %0, %1;" : "=f"(_abs_323) : "f"(pa)); - uint32_t _setp_321_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_321_bits) : "f"(_abs_322), "f"(_abs_323)); - bool _setp_321 = _setp_321_bits != 0; - float nm_err = ((_setp_321) ? nm_es : nm_ex); - float _add_324; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(c_a), "f"(nm_err)); - c_a = _add_324; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_338), "f"(s_a)); + float _abs_340; + asm("abs.f32 %0, %1;" : "=f"(_abs_340) : "f"(s_a)); + float _abs_341; + asm("abs.f32 %0, %1;" : "=f"(_abs_341) : "f"(pa)); + uint32_t _setp_339_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_339_bits) : "f"(_abs_340), "f"(_abs_341)); + bool _setp_339 = _setp_339_bits != 0; + float nm_err = ((_setp_339) ? nm_es : nm_ex); + float _add_342; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_342) : "f"(c_a), "f"(nm_err)); + c_a = _add_342; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_324), "f"(s_a)); + float _abs_326; + asm("abs.f32 %0, %1;" : "=f"(_abs_326) : "f"(s_a)); + float _abs_327; + asm("abs.f32 %0, %1;" : "=f"(_abs_327) : "f"(pa)); + uint32_t _setp_325_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_325_bits) : "f"(_abs_326), "f"(_abs_327)); + bool _setp_325 = _setp_325_bits != 0; + float nm_err = ((_setp_325) ? nm_es : nm_ex); + float _add_328; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(c_a), "f"(nm_err)); + c_a = _add_328; +#endif s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_325; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_325) : "f"(s_b), "f"(nm_t_0)); +#if __CUDA_ARCH__ == 1030 + float _sub_343; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_343) : "f"(s_b), "f"(nm_t_0)); +#else + float _sub_329; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_329) : "f"(s_b), "f"(nm_t_0)); +#endif float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_325), "f"(pb)); - float _sub_326; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_326) : "f"(pb), "f"(nm_t_0)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_343), "f"(pb)); + float _sub_344; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_344) : "f"(pb), "f"(nm_t_0)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_329), "f"(pb)); + float _sub_330; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_330) : "f"(pb), "f"(nm_t_0)); +#endif float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_326), "f"(s_b)); - float _abs_328; - asm("abs.f32 %0, %1;" : "=f"(_abs_328) : "f"(s_b)); - float _abs_329; - asm("abs.f32 %0, %1;" : "=f"(_abs_329) : "f"(pb)); - uint32_t _setp_327_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_327_bits) : "f"(_abs_328), "f"(_abs_329)); - bool _setp_327 = _setp_327_bits != 0; - float nm_err_0 = ((_setp_327) ? nm_es_0 : nm_ex_0); - float _add_330; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_330; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_344), "f"(s_b)); + float _abs_346; + asm("abs.f32 %0, %1;" : "=f"(_abs_346) : "f"(s_b)); + float _abs_347; + asm("abs.f32 %0, %1;" : "=f"(_abs_347) : "f"(pb)); + uint32_t _setp_345_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_345_bits) : "f"(_abs_346), "f"(_abs_347)); + bool _setp_345 = _setp_345_bits != 0; + float nm_err_0 = ((_setp_345) ? nm_es_0 : nm_ex_0); + float _add_348; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_348) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_348; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_330), "f"(s_b)); + float _abs_332; + asm("abs.f32 %0, %1;" : "=f"(_abs_332) : "f"(s_b)); + float _abs_333; + asm("abs.f32 %0, %1;" : "=f"(_abs_333) : "f"(pb)); + uint32_t _setp_331_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_331_bits) : "f"(_abs_332), "f"(_abs_333)); + bool _setp_331 = _setp_331_bits != 0; + float nm_err_0 = ((_setp_331) ? nm_es_0 : nm_ex_0); + float _add_334; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_334; +#endif s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4152,132 +4429,280 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_331 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_349 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_335 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_331), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_334; - asm("abs.f32 %0, %1;" : "=f"(_abs_334) : "f"(rp_z_3.x)); - float _neg_333; - asm("neg.f32 %0, %1;" : "=f"(_neg_333) : "f"(_abs_334)); - float _exp_332 = __nv_expf(_neg_333); - float _abs_337; - asm("abs.f32 %0, %1;" : "=f"(_abs_337) : "f"(rp_z_3.y)); - float _neg_336; - asm("neg.f32 %0, %1;" : "=f"(_neg_336) : "f"(_abs_337)); - float _exp_335 = __nv_expf(_neg_336); - float2 rp_e_3 = make_float2(_exp_332, _exp_335); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_349), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_352; + asm("abs.f32 %0, %1;" : "=f"(_abs_352) : "f"(rp_z_3.x)); + float _neg_351; + asm("neg.f32 %0, %1;" : "=f"(_neg_351) : "f"(_abs_352)); + float _exp_350 = __nv_expf(_neg_351); + float _abs_355; + asm("abs.f32 %0, %1;" : "=f"(_abs_355) : "f"(rp_z_3.y)); + float _neg_354; + asm("neg.f32 %0, %1;" : "=f"(_neg_354) : "f"(_abs_355)); + float _exp_353 = __nv_expf(_neg_354); + float2 rp_e_3 = make_float2(_exp_350, _exp_353); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_335), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_338; + asm("abs.f32 %0, %1;" : "=f"(_abs_338) : "f"(rp_z_3.x)); + float _neg_337; + asm("neg.f32 %0, %1;" : "=f"(_neg_337) : "f"(_abs_338)); + float _exp_336 = __nv_expf(_neg_337); + float _abs_341; + asm("abs.f32 %0, %1;" : "=f"(_abs_341) : "f"(rp_z_3.y)); + float _neg_340; + asm("neg.f32 %0, %1;" : "=f"(_neg_340) : "f"(_abs_341)); + float _exp_339 = __nv_expf(_neg_340); + float2 rp_e_3 = make_float2(_exp_336, _exp_339); +#endif float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_338 = approx_rcp(rp_den_3.x); - float _rcp_339 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_340 = make_float2(_rcp_338, _rcp_339); +#if __CUDA_ARCH__ == 1030 + float _rcp_356 = approx_rcp(rp_den_3.x); + float _rcp_357 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_358 = make_float2(_rcp_356, _rcp_357); +#else + float _rcp_342 = approx_rcp(rp_den_3.x); + float _rcp_343 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_344 = make_float2(_rcp_342, _rcp_343); +#endif float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_340)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_358)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_344)); +#endif float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_341 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_359 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_345 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_359)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_345)); +#endif u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_342 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_360 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_346 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_342), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_345; - asm("abs.f32 %0, %1;" : "=f"(_abs_345) : "f"(rp_z_4.x)); - float _neg_344; - asm("neg.f32 %0, %1;" : "=f"(_neg_344) : "f"(_abs_345)); - float _exp_343 = __nv_expf(_neg_344); - float _abs_348; - asm("abs.f32 %0, %1;" : "=f"(_abs_348) : "f"(rp_z_4.y)); - float _neg_347; - asm("neg.f32 %0, %1;" : "=f"(_neg_347) : "f"(_abs_348)); - float _exp_346 = __nv_expf(_neg_347); - float2 rp_e_4 = make_float2(_exp_343, _exp_346); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_360), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_363; + asm("abs.f32 %0, %1;" : "=f"(_abs_363) : "f"(rp_z_4.x)); + float _neg_362; + asm("neg.f32 %0, %1;" : "=f"(_neg_362) : "f"(_abs_363)); + float _exp_361 = __nv_expf(_neg_362); + float _abs_366; + asm("abs.f32 %0, %1;" : "=f"(_abs_366) : "f"(rp_z_4.y)); + float _neg_365; + asm("neg.f32 %0, %1;" : "=f"(_neg_365) : "f"(_abs_366)); + float _exp_364 = __nv_expf(_neg_365); + float2 rp_e_4 = make_float2(_exp_361, _exp_364); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_346), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_349; + asm("abs.f32 %0, %1;" : "=f"(_abs_349) : "f"(rp_z_4.x)); + float _neg_348; + asm("neg.f32 %0, %1;" : "=f"(_neg_348) : "f"(_abs_349)); + float _exp_347 = __nv_expf(_neg_348); + float _abs_352; + asm("abs.f32 %0, %1;" : "=f"(_abs_352) : "f"(rp_z_4.y)); + float _neg_351; + asm("neg.f32 %0, %1;" : "=f"(_neg_351) : "f"(_abs_352)); + float _exp_350 = __nv_expf(_neg_351); + float2 rp_e_4 = make_float2(_exp_347, _exp_350); +#endif float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_349 = approx_rcp(rp_den_4.x); - float _rcp_350 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_351 = make_float2(_rcp_349, _rcp_350); +#if __CUDA_ARCH__ == 1030 + float _rcp_367 = approx_rcp(rp_den_4.x); + float _rcp_368 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_369 = make_float2(_rcp_367, _rcp_368); +#else + float _rcp_353 = approx_rcp(rp_den_4.x); + float _rcp_354 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_355 = make_float2(_rcp_353, _rcp_354); +#endif float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_351)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_369)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_355)); +#endif float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_352 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_370 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_356 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_352)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_370)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_356)); +#endif u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_353 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_371 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_357 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_356; - asm("abs.f32 %0, %1;" : "=f"(_abs_356) : "f"(rp_z_5.x)); - float _neg_355; - asm("neg.f32 %0, %1;" : "=f"(_neg_355) : "f"(_abs_356)); - float _exp_354 = __nv_expf(_neg_355); - float _abs_359; - asm("abs.f32 %0, %1;" : "=f"(_abs_359) : "f"(rp_z_5.y)); - float _neg_358; - asm("neg.f32 %0, %1;" : "=f"(_neg_358) : "f"(_abs_359)); - float _exp_357 = __nv_expf(_neg_358); - float2 rp_e_5 = make_float2(_exp_354, _exp_357); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_371), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_374; + asm("abs.f32 %0, %1;" : "=f"(_abs_374) : "f"(rp_z_5.x)); + float _neg_373; + asm("neg.f32 %0, %1;" : "=f"(_neg_373) : "f"(_abs_374)); + float _exp_372 = __nv_expf(_neg_373); + float _abs_377; + asm("abs.f32 %0, %1;" : "=f"(_abs_377) : "f"(rp_z_5.y)); + float _neg_376; + asm("neg.f32 %0, %1;" : "=f"(_neg_376) : "f"(_abs_377)); + float _exp_375 = __nv_expf(_neg_376); + float2 rp_e_5 = make_float2(_exp_372, _exp_375); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_357), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_360; + asm("abs.f32 %0, %1;" : "=f"(_abs_360) : "f"(rp_z_5.x)); + float _neg_359; + asm("neg.f32 %0, %1;" : "=f"(_neg_359) : "f"(_abs_360)); + float _exp_358 = __nv_expf(_neg_359); + float _abs_363; + asm("abs.f32 %0, %1;" : "=f"(_abs_363) : "f"(rp_z_5.y)); + float _neg_362; + asm("neg.f32 %0, %1;" : "=f"(_neg_362) : "f"(_abs_363)); + float _exp_361 = __nv_expf(_neg_362); + float2 rp_e_5 = make_float2(_exp_358, _exp_361); +#endif float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_360 = approx_rcp(rp_den_5.x); - float _rcp_361 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_362 = make_float2(_rcp_360, _rcp_361); +#if __CUDA_ARCH__ == 1030 + float _rcp_378 = approx_rcp(rp_den_5.x); + float _rcp_379 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_380 = make_float2(_rcp_378, _rcp_379); +#else + float _rcp_364 = approx_rcp(rp_den_5.x); + float _rcp_365 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_366 = make_float2(_rcp_364, _rcp_365); +#endif float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_362)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_380)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_366)); +#endif float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_363 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_381 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_367 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_363)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_381)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_367)); +#endif u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_364 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_382 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_368 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_364), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_367; - asm("abs.f32 %0, %1;" : "=f"(_abs_367) : "f"(rp_z_6.x)); - float _neg_366; - asm("neg.f32 %0, %1;" : "=f"(_neg_366) : "f"(_abs_367)); - float _exp_365 = __nv_expf(_neg_366); - float _abs_370; - asm("abs.f32 %0, %1;" : "=f"(_abs_370) : "f"(rp_z_6.y)); - float _neg_369; - asm("neg.f32 %0, %1;" : "=f"(_neg_369) : "f"(_abs_370)); - float _exp_368 = __nv_expf(_neg_369); - float2 rp_e_6 = make_float2(_exp_365, _exp_368); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_382), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_385; + asm("abs.f32 %0, %1;" : "=f"(_abs_385) : "f"(rp_z_6.x)); + float _neg_384; + asm("neg.f32 %0, %1;" : "=f"(_neg_384) : "f"(_abs_385)); + float _exp_383 = __nv_expf(_neg_384); + float _abs_388; + asm("abs.f32 %0, %1;" : "=f"(_abs_388) : "f"(rp_z_6.y)); + float _neg_387; + asm("neg.f32 %0, %1;" : "=f"(_neg_387) : "f"(_abs_388)); + float _exp_386 = __nv_expf(_neg_387); + float2 rp_e_6 = make_float2(_exp_383, _exp_386); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_368), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_371; + asm("abs.f32 %0, %1;" : "=f"(_abs_371) : "f"(rp_z_6.x)); + float _neg_370; + asm("neg.f32 %0, %1;" : "=f"(_neg_370) : "f"(_abs_371)); + float _exp_369 = __nv_expf(_neg_370); + float _abs_374; + asm("abs.f32 %0, %1;" : "=f"(_abs_374) : "f"(rp_z_6.y)); + float _neg_373; + asm("neg.f32 %0, %1;" : "=f"(_neg_373) : "f"(_abs_374)); + float _exp_372 = __nv_expf(_neg_373); + float2 rp_e_6 = make_float2(_exp_369, _exp_372); +#endif float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_371 = approx_rcp(rp_den_6.x); - float _rcp_372 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_373 = make_float2(_rcp_371, _rcp_372); +#if __CUDA_ARCH__ == 1030 + float _rcp_389 = approx_rcp(rp_den_6.x); + float _rcp_390 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_391 = make_float2(_rcp_389, _rcp_390); +#else + float _rcp_375 = approx_rcp(rp_den_6.x); + float _rcp_376 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_377 = make_float2(_rcp_375, _rcp_376); +#endif float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_373)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_391)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_377)); +#endif float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_374 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_392 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_378 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_374)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_392)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_378)); +#endif u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4292,54 +4717,102 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_375; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_375) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_375; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_393; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_393) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_393; +#else + uint32_t _prmt_379; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_379; +#endif int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_376 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_376)); - *(reinterpret_cast(_gst_376) + (0)) = pq_4[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_394 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_394)); + *(reinterpret_cast(_gst_394) + (0)) = pq_4[0]; +#else + auto _gst_380 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = pq_4[0]; +#endif } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_377; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_377) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_377; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_395; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_395) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_395; +#else + uint32_t _prmt_381; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_381; +#endif int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_378 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_378)); - *(reinterpret_cast(_gst_378) + (0)) = pq_5[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_396 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_396)); + *(reinterpret_cast(_gst_396) + (0)) = pq_5[0]; +#else + auto _gst_382 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = pq_5[0]; +#endif } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_379; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_379; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_397; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_397) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_397; +#else + uint32_t _prmt_383; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_383) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_383; +#endif int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_380 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_380)); - *(reinterpret_cast(_gst_380) + (0)) = pq_6[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_398 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_398)); + *(reinterpret_cast(_gst_398) + (0)) = pq_6[0]; +#else + auto _gst_384 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = pq_6[0]; +#endif } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_381; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_381; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_399; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_399) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_399; +#else + uint32_t _prmt_385; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_385) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_385; +#endif int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_382 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_382)); - *(reinterpret_cast(_gst_382) + (0)) = pq_7[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_400 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_400)); + *(reinterpret_cast(_gst_400) + (0)) = pq_7[0]; +#else + auto _gst_386 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_386)); + *(reinterpret_cast(_gst_386) + (0)) = pq_7[0]; +#endif } } int t_a_8 = 32 + 8 * th; @@ -4404,47 +4877,99 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_383; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_383) : "f"(s_a_0), "f"(nm_t_1)); +#if __CUDA_ARCH__ == 1030 + float _sub_401; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_401) : "f"(s_a_0), "f"(nm_t_1)); +#else + float _sub_387; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_387) : "f"(s_a_0), "f"(nm_t_1)); +#endif float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_383), "f"(pa_0)); - float _sub_384; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_384) : "f"(pa_0), "f"(nm_t_1)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_401), "f"(pa_0)); + float _sub_402; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_402) : "f"(pa_0), "f"(nm_t_1)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_387), "f"(pa_0)); + float _sub_388; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_388) : "f"(pa_0), "f"(nm_t_1)); +#endif float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_384), "f"(s_a_0)); - float _abs_386; - asm("abs.f32 %0, %1;" : "=f"(_abs_386) : "f"(s_a_0)); - float _abs_387; - asm("abs.f32 %0, %1;" : "=f"(_abs_387) : "f"(pa_0)); - uint32_t _setp_385_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_385_bits) : "f"(_abs_386), "f"(_abs_387)); - bool _setp_385 = _setp_385_bits != 0; - float nm_err_1 = ((_setp_385) ? nm_es_1 : nm_ex_1); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_388; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_402), "f"(s_a_0)); + float _abs_404; + asm("abs.f32 %0, %1;" : "=f"(_abs_404) : "f"(s_a_0)); + float _abs_405; + asm("abs.f32 %0, %1;" : "=f"(_abs_405) : "f"(pa_0)); + uint32_t _setp_403_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_403_bits) : "f"(_abs_404), "f"(_abs_405)); + bool _setp_403 = _setp_403_bits != 0; + float nm_err_1 = ((_setp_403) ? nm_es_1 : nm_ex_1); + float _add_406; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_406) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_406; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_388), "f"(s_a_0)); + float _abs_390; + asm("abs.f32 %0, %1;" : "=f"(_abs_390) : "f"(s_a_0)); + float _abs_391; + asm("abs.f32 %0, %1;" : "=f"(_abs_391) : "f"(pa_0)); + uint32_t _setp_389_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_389_bits) : "f"(_abs_390), "f"(_abs_391)); + bool _setp_389 = _setp_389_bits != 0; + float nm_err_1 = ((_setp_389) ? nm_es_1 : nm_ex_1); + float _add_392; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_392; +#endif s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_389; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_389) : "f"(s_b_0), "f"(nm_t_2)); +#if __CUDA_ARCH__ == 1030 + float _sub_407; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_407) : "f"(s_b_0), "f"(nm_t_2)); +#else + float _sub_393; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_393) : "f"(s_b_0), "f"(nm_t_2)); +#endif float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_389), "f"(pb_0)); - float _sub_390; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_390) : "f"(pb_0), "f"(nm_t_2)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_407), "f"(pb_0)); + float _sub_408; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_408) : "f"(pb_0), "f"(nm_t_2)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_393), "f"(pb_0)); + float _sub_394; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_394) : "f"(pb_0), "f"(nm_t_2)); +#endif float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_390), "f"(s_b_0)); - float _abs_392; - asm("abs.f32 %0, %1;" : "=f"(_abs_392) : "f"(s_b_0)); - float _abs_393; - asm("abs.f32 %0, %1;" : "=f"(_abs_393) : "f"(pb_0)); - uint32_t _setp_391_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_391_bits) : "f"(_abs_392), "f"(_abs_393)); - bool _setp_391 = _setp_391_bits != 0; - float nm_err_2 = ((_setp_391) ? nm_es_2 : nm_ex_2); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_394; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_408), "f"(s_b_0)); + float _abs_410; + asm("abs.f32 %0, %1;" : "=f"(_abs_410) : "f"(s_b_0)); + float _abs_411; + asm("abs.f32 %0, %1;" : "=f"(_abs_411) : "f"(pb_0)); + uint32_t _setp_409_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_409_bits) : "f"(_abs_410), "f"(_abs_411)); + bool _setp_409 = _setp_409_bits != 0; + float nm_err_2 = ((_setp_409) ? nm_es_2 : nm_ex_2); + float _add_412; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_412) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_412; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_394), "f"(s_b_0)); + float _abs_396; + asm("abs.f32 %0, %1;" : "=f"(_abs_396) : "f"(s_b_0)); + float _abs_397; + asm("abs.f32 %0, %1;" : "=f"(_abs_397) : "f"(pb_0)); + uint32_t _setp_395_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_395_bits) : "f"(_abs_396), "f"(_abs_397)); + bool _setp_395 = _setp_395_bits != 0; + float nm_err_2 = ((_setp_395) ? nm_es_2 : nm_ex_2); + float _add_398; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_398; +#endif s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4452,8 +4977,13 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_395 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_395), "f"(off_acc[0]) : "memory"); +#if __CUDA_ARCH__ == 1030 + uint32_t _addr_413 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_413), "f"(off_acc[0]) : "memory"); +#else + uint32_t _addr_399 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_399), "f"(off_acc[0]) : "memory"); +#endif } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4479,9 +5009,15 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1 = r_297[0]; +#if __CUDA_ARCH__ == 1030 + float r_276[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1 = r_276[0]; +#else + float r_255[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1 = r_255[0]; +#endif float gc_2 = gn_1 - 21.640425f; int eb_1 = 81920 + th * -32768; int ew_1 = eb_1 >> 2; @@ -4490,368 +5026,831 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int gb8_1 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1 * 128); int kb8_1 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1 * 128); float dbp_1[8]; - float qo_1[8]; - tmem_ld_x8(&qo_1[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); - float ko_1[8]; - tmem_ld_x8(&ko_1[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); - float qdg_1[8]; - tmem_ld_x8(&qdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); - float kdg_1[8]; - tmem_ld_x8(&kdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); +#if __CUDA_ARCH__ == 1030 + float qo_0[8]; + tmem_ld_x8(&qo_0[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); + float ko_0[8]; + tmem_ld_x8(&ko_0[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); + float qdg_0[8]; + tmem_ld_x8(&qdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); + float kdg_0[8]; + tmem_ld_x8(&kdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); +#else + float qo_2[8]; + tmem_ld_x8(&qo_2[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); + float ko_2[8]; + tmem_ld_x8(&ko_2[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); + float qdg_2[8]; + tmem_ld_x8(&qdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); + float kdg_2[8]; + tmem_ld_x8(&kdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); +#endif float dkt_2[8]; tmem_ld_x8(&dkt_2[0], taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th)); - float btv_1[8]; +#if __CUDA_ARCH__ == 1030 + float btv_0[8]; +#else + float btv_2[8]; +#endif asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1[0])), "=r"(*reinterpret_cast(&btv_1[(0) + 1])), "=r"(*reinterpret_cast(&btv_1[(0) + 2])), "=r"(*reinterpret_cast(&btv_1[(0) + 3])) +#if __CUDA_ARCH__ == 1030 + : "=r"(*reinterpret_cast(&btv_0[0])), "=r"(*reinterpret_cast(&btv_0[(0) + 1])), "=r"(*reinterpret_cast(&btv_0[(0) + 2])), "=r"(*reinterpret_cast(&btv_0[(0) + 3])) +#else + : "=r"(*reinterpret_cast(&btv_2[0])), "=r"(*reinterpret_cast(&btv_2[(0) + 1])), "=r"(*reinterpret_cast(&btv_2[(0) + 2])), "=r"(*reinterpret_cast(&btv_2[(0) + 3])) +#endif : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1[4])), "=r"(*reinterpret_cast(&btv_1[(4) + 1])), "=r"(*reinterpret_cast(&btv_1[(4) + 2])), "=r"(*reinterpret_cast(&btv_1[(4) + 3])) +#if __CUDA_ARCH__ == 1030 + : "=r"(*reinterpret_cast(&btv_0[4])), "=r"(*reinterpret_cast(&btv_0[(4) + 1])), "=r"(*reinterpret_cast(&btv_0[(4) + 2])), "=r"(*reinterpret_cast(&btv_0[(4) + 3])) +#else + : "=r"(*reinterpret_cast(&btv_2[4])), "=r"(*reinterpret_cast(&btv_2[(4) + 1])), "=r"(*reinterpret_cast(&btv_2[(4) + 2])), "=r"(*reinterpret_cast(&btv_2[(4) + 3])) +#endif : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th) + 16))); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1[1]; s_run_1[0] = 0.0f; - int t_16 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1 + 2944 + d_lane]; - float bt_t_16 = btv_1[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1); - float e_dg_16 = approx_exp2(g_t_16 - gc_2); - float dq2_16 = __fmaf_rn(qo_1[7], e_off_16, qdg_1[7] * e_dg_16); - float dk2_16 = __fmaf_rn(ko_1[7], e_off_16, kdg_1[7] * e_dg_16); - dbp_1[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + dkt_2[7]; - float dg_o_16 = __fmaf_rn(q_t_1_16, dq2_16, (dk2b_16 - dkt_2[7]) * k_t_16) + dgf_16; - s_run_1[0] = s_run_1[0] + dg_o_16; +#if __CUDA_ARCH__ == 1030 + int t_8 = 16 + 8 * th + 7; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_8 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_8 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_8 = __uint_as_float(r_279[0] << 16); + float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); + float bt_t_8 = btv_0[7]; + float e_off_8 = approx_exp2(g_t_8 - gn_1); + float e_dg_8 = approx_exp2(g_t_8 - gc_2); + float dq2_8 = __fmaf_rn(qo_0[7], e_off_8, qdg_0[7] * e_dg_8); + float dk2_8 = __fmaf_rn(ko_0[7], e_off_8, kdg_0[7] * e_dg_8); + dbp_1[7] = dk2_8 * k_t_8; + float dk2b_8 = dk2_8 * bt_t_8; + float qe_7 = q_t_1_8 * e_dg_8; + float kp_19 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_8) * kdg_0[7]); + float dqf_8 = pool_f32[ew_1 + 896 + d_lane]; + float dkf_8 = pool_f32[ew_1 + 1920 + d_lane]; + float dgf_8 = pool_f32[ew_1 + 2944 + d_lane]; + float dq_o_8 = dq2_8 + dqf_8; + float dk_o_8 = dk2b_8 + dkf_8 + dkt_2[7]; + float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_2[7], k_t_8, cr_7)) + dgf_8; + s_run_1[0] = s_run_1[0] + dg_o_8; +#else + int t_0 = 16 + 8 * th + 7; + float r_256[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_0 = r_256[0]; + unsigned int r_257[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_0 = __uint_as_float(r_257[0] << 16); + unsigned int r_258[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_0 = __uint_as_float(r_258[0] << 16); + float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); + float bt_t_0 = btv_2[7]; + float e_off_0 = approx_exp2(g_t_0 - gn_1); + float e_dg_0 = approx_exp2(g_t_0 - gc_2); + float dq2_0 = __fmaf_rn(qo_2[7], e_off_0, qdg_2[7] * e_dg_0); + float dk2_0 = __fmaf_rn(ko_2[7], e_off_0, kdg_2[7] * e_dg_0); + dbp_1[7] = dk2_0 * k_t_0; + float dk2b_0 = dk2_0 * bt_t_0; + float qe_2 = q_t_1_0 * e_dg_0; + float kp_13 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_13; + float kbe_2 = (float)kb_2 * e_dg_0; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg_2[7], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_13 * e_dg_0) * kdg_2[7]); + float dqf_0 = pool_f32[ew_1 + 896 + d_lane]; + float dkf_0 = pool_f32[ew_1 + 1920 + d_lane]; + float dgf_0 = pool_f32[ew_1 + 2944 + d_lane]; + float dq_o_0 = dq2_0 + dqf_0; + float dk_o_0 = dk2b_0 + dkf_0 + dkt_2[7]; + float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_2[7], k_t_0, cr_2)) + dgf_0; + s_run_1[0] = s_run_1[0] + dg_o_0; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_16 = t_16 * qrs; - if (t_16 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_8 = t_8 * qrs; + if (t_8 < clen) { +#else + int qoff_0 = t_0 * qrs; + if (t_0 < clen) { +#endif { - auto _gst_396 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_396)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_396) + (0)) = (__nv_bfloat16)dq_o_16; +#if __CUDA_ARCH__ == 1030 + auto _gst_414 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_414)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_414) + (0)) = (__nv_bfloat16)dq_o_8; +#else + auto _gst_400 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_400)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_0; +#endif } { - auto _gst_397 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_397)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_397) + (0)) = (__nv_bfloat16)dk_o_16; +#if __CUDA_ARCH__ == 1030 + auto _gst_415 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_415)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_415) + (0)) = (__nv_bfloat16)dk_o_8; +#else + auto _gst_401 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_401)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_0; +#endif } } - int t_17 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1 + 2816 + d_lane]; - float bt_t_17 = btv_1[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1); - float e_dg_17 = approx_exp2(g_t_17 - gc_2); - float dq2_17 = __fmaf_rn(qo_1[6], e_off_17, qdg_1[6] * e_dg_17); - float dk2_17 = __fmaf_rn(ko_1[6], e_off_17, kdg_1[6] * e_dg_17); - dbp_1[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + dkt_2[6]; - float dg_o_17 = __fmaf_rn(q_t_1_17, dq2_17, (dk2b_17 - dkt_2[6]) * k_t_17) + dgf_17; - s_run_1[0] = s_run_1[0] + dg_o_17; +#if __CUDA_ARCH__ == 1030 + int t_9 = 16 + 8 * th + 6; + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_9 = r_280[0]; + unsigned int r_281[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_9 = __uint_as_float(r_281[0] << 16); + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_9 = __uint_as_float(r_282[0] << 16); + float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); + float bt_t_9 = btv_0[6]; + float e_off_9 = approx_exp2(g_t_9 - gn_1); + float e_dg_9 = approx_exp2(g_t_9 - gc_2); + float dq2_9 = __fmaf_rn(qo_0[6], e_off_9, qdg_0[6] * e_dg_9); + float dk2_9 = __fmaf_rn(ko_0[6], e_off_9, kdg_0[6] * e_dg_9); + dbp_1[6] = dk2_9 * k_t_9; + float dk2b_9 = dk2_9 * bt_t_9; + float qe_8 = q_t_1_9 * e_dg_9; + float kp_20 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_9) * kdg_0[6]); + float dqf_9 = pool_f32[ew_1 + 768 + d_lane]; + float dkf_9 = pool_f32[ew_1 + 1792 + d_lane]; + float dgf_9 = pool_f32[ew_1 + 2816 + d_lane]; + float dq_o_9 = dq2_9 + dqf_9; + float dk_o_9 = dk2b_9 + dkf_9 + dkt_2[6]; + float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_2[6], k_t_9, cr_8)) + dgf_9; + s_run_1[0] = s_run_1[0] + dg_o_9; +#else + int t_1_1 = 16 + 8 * th + 6; + float r_259[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1_1 = r_259[0]; + unsigned int r_260[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1_1 = __uint_as_float(r_260[0] << 16); + unsigned int r_261[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1_2 = __uint_as_float(r_261[0] << 16); + float q_t_1_1_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_2); + float bt_t_1_1 = btv_2[6]; + float e_off_1_1 = approx_exp2(g_t_1_1 - gn_1); + float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_2); + float dq2_1_1 = __fmaf_rn(qo_2[6], e_off_1_1, qdg_2[6] * e_dg_1_1); + float dk2_1_1 = __fmaf_rn(ko_2[6], e_off_1_1, kdg_2[6] * e_dg_1_1); + dbp_1[6] = dk2_1_1 * k_t_1_1; + float dk2b_1_1 = dk2_1_1 * bt_t_1_1; + float qe_0 = q_t_1_1_1 * e_dg_1_1; + float kp_12_1 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12_1; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg_2[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12_1 * e_dg_1_1) * kdg_2[6]); + float dqf_1_1 = pool_f32[ew_1 + 768 + d_lane]; + float dkf_1_1 = pool_f32[ew_1 + 1792 + d_lane]; + float dgf_1_1 = pool_f32[ew_1 + 2816 + d_lane]; + float dq_o_1_1 = dq2_1_1 + dqf_1_1; + float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + dkt_2[6]; + float dg_o_1_1 = __fmaf_rn(q_t_1_1_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - dkt_2[6], k_t_1_1, cr_0)) + dgf_1_1; + s_run_1[0] = s_run_1[0] + dg_o_1_1; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_17 = t_17 * qrs; - if (t_17 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_9 = t_9 * qrs; + if (t_9 < clen) { +#else + int qoff_1_1 = t_1_1 * qrs; + if (t_1_1 < clen) { +#endif { - auto _gst_398 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_398)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_398) + (0)) = (__nv_bfloat16)dq_o_17; +#if __CUDA_ARCH__ == 1030 + auto _gst_416 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_416)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_416) + (0)) = (__nv_bfloat16)dq_o_9; +#else + auto _gst_402 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_402)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_1_1; +#endif } { - auto _gst_399 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_399)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_399) + (0)) = (__nv_bfloat16)dk_o_17; +#if __CUDA_ARCH__ == 1030 + auto _gst_417 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_417)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_417) + (0)) = (__nv_bfloat16)dk_o_9; +#else + auto _gst_403 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_403)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_1_1; +#endif } } - int t_18 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1 + 2688 + d_lane]; - float bt_t_18 = btv_1[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1); - float e_dg_18 = approx_exp2(g_t_18 - gc_2); - float dq2_18 = __fmaf_rn(qo_1[5], e_off_18, qdg_1[5] * e_dg_18); - float dk2_18 = __fmaf_rn(ko_1[5], e_off_18, kdg_1[5] * e_dg_18); - dbp_1[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + dkt_2[5]; - float dg_o_18 = __fmaf_rn(q_t_1_18, dq2_18, (dk2b_18 - dkt_2[5]) * k_t_18) + dgf_18; - s_run_1[0] = s_run_1[0] + dg_o_18; +#if __CUDA_ARCH__ == 1030 + int t_10 = 16 + 8 * th + 5; + float r_283[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_10 = r_283[0]; + unsigned int r_284[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_10 = __uint_as_float(r_284[0] << 16); + unsigned int r_285[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_10 = __uint_as_float(r_285[0] << 16); + float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); + float bt_t_10 = btv_0[5]; + float e_off_10 = approx_exp2(g_t_10 - gn_1); + float e_dg_10 = approx_exp2(g_t_10 - gc_2); + float dq2_10 = __fmaf_rn(qo_0[5], e_off_10, qdg_0[5] * e_dg_10); + float dk2_10 = __fmaf_rn(ko_0[5], e_off_10, kdg_0[5] * e_dg_10); + dbp_1[5] = dk2_10 * k_t_10; + float dk2b_10 = dk2_10 * bt_t_10; + float qe_9 = q_t_1_10 * e_dg_10; + float kp_21 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_10) * kdg_0[5]); + float dqf_10 = pool_f32[ew_1 + 640 + d_lane]; + float dkf_10 = pool_f32[ew_1 + 1664 + d_lane]; + float dgf_10 = pool_f32[ew_1 + 2688 + d_lane]; + float dq_o_10 = dq2_10 + dqf_10; + float dk_o_10 = dk2b_10 + dkf_10 + dkt_2[5]; + float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_2[5], k_t_10, cr_9)) + dgf_10; + s_run_1[0] = s_run_1[0] + dg_o_10; +#else + int t_2 = 16 + 8 * th + 5; + float r_262[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_262[0]; + unsigned int r_263[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_263[0] << 16); + unsigned int r_264[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2_1 = __uint_as_float(r_264[0] << 16); + float q_t_1_2_1 = ((t_2 >= clen) ? 0.0f : q_t_2_1); + float bt_t_2 = btv_2[5]; + float e_off_2 = approx_exp2(g_t_2 - gn_1); + float e_dg_2 = approx_exp2(g_t_2 - gc_2); + float dq2_2 = __fmaf_rn(qo_2[5], e_off_2, qdg_2[5] * e_dg_2); + float dk2_2 = __fmaf_rn(ko_2[5], e_off_2, kdg_2[5] * e_dg_2); + dbp_1[5] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float qe_1_1 = q_t_1_2_1 * e_dg_2; + float kp_13_1 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1_1 = (__nv_bfloat16)kp_13_1; + float kbe_1_1 = (float)kb_1_1 * e_dg_2; + float cr_1_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1_1) & 4294959104u) - qe_1_1, qdg_2[5], (__uint_as_float(__as_u32(kbe_1_1) & 4294959104u) - kp_13_1 * e_dg_2) * kdg_2[5]); + float dqf_2 = pool_f32[ew_1 + 640 + d_lane]; + float dkf_2 = pool_f32[ew_1 + 1664 + d_lane]; + float dgf_2 = pool_f32[ew_1 + 2688 + d_lane]; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + dkt_2[5]; + float dg_o_2 = __fmaf_rn(q_t_1_2_1, dq2_2, __fmaf_rn(dk2b_2 - dkt_2[5], k_t_2, cr_1_1)) + dgf_2; + s_run_1[0] = s_run_1[0] + dg_o_2; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_18 = t_18 * qrs; - if (t_18 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_10 = t_10 * qrs; + if (t_10 < clen) { +#else + int qoff_2 = t_2 * qrs; + if (t_2 < clen) { +#endif { - auto _gst_400 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_400)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_18; +#if __CUDA_ARCH__ == 1030 + auto _gst_418 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_418)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_418) + (0)) = (__nv_bfloat16)dq_o_10; +#else + auto _gst_404 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_404)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_2; +#endif } { - auto _gst_401 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_401)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_18; +#if __CUDA_ARCH__ == 1030 + auto _gst_419 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_419)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_419) + (0)) = (__nv_bfloat16)dk_o_10; +#else + auto _gst_405 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_405)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_2; +#endif } } - int t_19 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1 + 2560 + d_lane]; - float bt_t_19 = btv_1[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1); - float e_dg_19 = approx_exp2(g_t_19 - gc_2); - float dq2_19 = __fmaf_rn(qo_1[4], e_off_19, qdg_1[4] * e_dg_19); - float dk2_19 = __fmaf_rn(ko_1[4], e_off_19, kdg_1[4] * e_dg_19); - dbp_1[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + dkt_2[4]; - float dg_o_19 = __fmaf_rn(q_t_1_19, dq2_19, (dk2b_19 - dkt_2[4]) * k_t_19) + dgf_19; - s_run_1[0] = s_run_1[0] + dg_o_19; +#if __CUDA_ARCH__ == 1030 + int t_11 = 16 + 8 * th + 4; + float r_286[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_11 = r_286[0]; + unsigned int r_287[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_11 = __uint_as_float(r_287[0] << 16); + unsigned int r_288[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_11 = __uint_as_float(r_288[0] << 16); + float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); + float bt_t_11 = btv_0[4]; + float e_off_11 = approx_exp2(g_t_11 - gn_1); + float e_dg_11 = approx_exp2(g_t_11 - gc_2); + float dq2_11 = __fmaf_rn(qo_0[4], e_off_11, qdg_0[4] * e_dg_11); + float dk2_11 = __fmaf_rn(ko_0[4], e_off_11, kdg_0[4] * e_dg_11); + dbp_1[4] = dk2_11 * k_t_11; + float dk2b_11 = dk2_11 * bt_t_11; + float qe_10 = q_t_1_11 * e_dg_11; + float kp_22 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_11) * kdg_0[4]); + float dqf_11 = pool_f32[ew_1 + 512 + d_lane]; + float dkf_11 = pool_f32[ew_1 + 1536 + d_lane]; + float dgf_11 = pool_f32[ew_1 + 2560 + d_lane]; + float dq_o_11 = dq2_11 + dqf_11; + float dk_o_11 = dk2b_11 + dkf_11 + dkt_2[4]; + float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_2[4], k_t_11, cr_10)) + dgf_11; + s_run_1[0] = s_run_1[0] + dg_o_11; +#else + int t_3 = 16 + 8 * th + 4; + float r_265[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_3 = r_265[0]; + unsigned int r_266[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_3 = __uint_as_float(r_266[0] << 16); + unsigned int r_267[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_3 = __uint_as_float(r_267[0] << 16); + float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); + float bt_t_3 = btv_2[4]; + float e_off_3 = approx_exp2(g_t_3 - gn_1); + float e_dg_3 = approx_exp2(g_t_3 - gc_2); + float dq2_3 = __fmaf_rn(qo_2[4], e_off_3, qdg_2[4] * e_dg_3); + float dk2_3 = __fmaf_rn(ko_2[4], e_off_3, kdg_2[4] * e_dg_3); + dbp_1[4] = dk2_3 * k_t_3; + float dk2b_3 = dk2_3 * bt_t_3; + float qe_2_1 = q_t_1_3 * e_dg_3; + float kp_14 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2_1 = (__nv_bfloat16)kp_14; + float kbe_2_1 = (float)kb_2_1 * e_dg_3; + float cr_2_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_2_1) & 4294959104u) - qe_2_1, qdg_2[4], (__uint_as_float(__as_u32(kbe_2_1) & 4294959104u) - kp_14 * e_dg_3) * kdg_2[4]); + float dqf_3 = pool_f32[ew_1 + 512 + d_lane]; + float dkf_3 = pool_f32[ew_1 + 1536 + d_lane]; + float dgf_3 = pool_f32[ew_1 + 2560 + d_lane]; + float dq_o_3 = dq2_3 + dqf_3; + float dk_o_3 = dk2b_3 + dkf_3 + dkt_2[4]; + float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_2[4], k_t_3, cr_2_1)) + dgf_3; + s_run_1[0] = s_run_1[0] + dg_o_3; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_19 = t_19 * qrs; - if (t_19 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_11 = t_11 * qrs; + if (t_11 < clen) { +#else + int qoff_3 = t_3 * qrs; + if (t_3 < clen) { +#endif { - auto _gst_402 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_402)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_19; +#if __CUDA_ARCH__ == 1030 + auto _gst_420 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_420)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_420) + (0)) = (__nv_bfloat16)dq_o_11; +#else + auto _gst_406 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_406)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_3; +#endif } { - auto _gst_403 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_403)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_19; +#if __CUDA_ARCH__ == 1030 + auto _gst_421 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_421)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_421) + (0)) = (__nv_bfloat16)dk_o_11; +#else + auto _gst_407 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_407)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_3; +#endif } } - int t_20 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1 + 2432 + d_lane]; - float bt_t_20 = btv_1[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1); - float e_dg_20 = approx_exp2(g_t_20 - gc_2); - float dq2_20 = __fmaf_rn(qo_1[3], e_off_20, qdg_1[3] * e_dg_20); - float dk2_20 = __fmaf_rn(ko_1[3], e_off_20, kdg_1[3] * e_dg_20); - dbp_1[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + dkt_2[3]; - float dg_o_20 = __fmaf_rn(q_t_1_20, dq2_20, (dk2b_20 - dkt_2[3]) * k_t_20) + dgf_20; - s_run_1[0] = s_run_1[0] + dg_o_20; +#if __CUDA_ARCH__ == 1030 + int t_12 = 16 + 8 * th + 3; + float r_289[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_12 = r_289[0]; + unsigned int r_290[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_12 = __uint_as_float(r_290[0] << 16); + unsigned int r_291[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_12 = __uint_as_float(r_291[0] << 16); + float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); + float bt_t_12 = btv_0[3]; + float e_off_12 = approx_exp2(g_t_12 - gn_1); + float e_dg_12 = approx_exp2(g_t_12 - gc_2); + float dq2_12 = __fmaf_rn(qo_0[3], e_off_12, qdg_0[3] * e_dg_12); + float dk2_12 = __fmaf_rn(ko_0[3], e_off_12, kdg_0[3] * e_dg_12); + dbp_1[3] = dk2_12 * k_t_12; + float dk2b_12 = dk2_12 * bt_t_12; + float qe_11 = q_t_1_12 * e_dg_12; + float kp_23 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_12) * kdg_0[3]); + float dqf_12 = pool_f32[ew_1 + 384 + d_lane]; + float dkf_12 = pool_f32[ew_1 + 1408 + d_lane]; + float dgf_12 = pool_f32[ew_1 + 2432 + d_lane]; + float dq_o_12 = dq2_12 + dqf_12; + float dk_o_12 = dk2b_12 + dkf_12 + dkt_2[3]; + float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_2[3], k_t_12, cr_11)) + dgf_12; + s_run_1[0] = s_run_1[0] + dg_o_12; +#else + int t_4 = 16 + 8 * th + 3; + float r_268[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_4 = r_268[0]; + unsigned int r_269[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_4 = __uint_as_float(r_269[0] << 16); + unsigned int r_270[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_4 = __uint_as_float(r_270[0] << 16); + float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); + float bt_t_4 = btv_2[3]; + float e_off_4 = approx_exp2(g_t_4 - gn_1); + float e_dg_4 = approx_exp2(g_t_4 - gc_2); + float dq2_4 = __fmaf_rn(qo_2[3], e_off_4, qdg_2[3] * e_dg_4); + float dk2_4 = __fmaf_rn(ko_2[3], e_off_4, kdg_2[3] * e_dg_4); + dbp_1[3] = dk2_4 * k_t_4; + float dk2b_4 = dk2_4 * bt_t_4; + float qe_3 = q_t_1_4 * e_dg_4; + float kp_15 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg_2[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_4) * kdg_2[3]); + float dqf_4 = pool_f32[ew_1 + 384 + d_lane]; + float dkf_4 = pool_f32[ew_1 + 1408 + d_lane]; + float dgf_4 = pool_f32[ew_1 + 2432 + d_lane]; + float dq_o_4 = dq2_4 + dqf_4; + float dk_o_4 = dk2b_4 + dkf_4 + dkt_2[3]; + float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_2[3], k_t_4, cr_3)) + dgf_4; + s_run_1[0] = s_run_1[0] + dg_o_4; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_20 = t_20 * qrs; - if (t_20 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_12 = t_12 * qrs; + if (t_12 < clen) { +#else + int qoff_4 = t_4 * qrs; + if (t_4 < clen) { +#endif { - auto _gst_404 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_404)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_20; +#if __CUDA_ARCH__ == 1030 + auto _gst_422 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_422)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_422) + (0)) = (__nv_bfloat16)dq_o_12; +#else + auto _gst_408 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_408)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_4; +#endif } { - auto _gst_405 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_405)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_20; +#if __CUDA_ARCH__ == 1030 + auto _gst_423 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_423)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_423) + (0)) = (__nv_bfloat16)dk_o_12; +#else + auto _gst_409 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_409)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_4; +#endif } } - int t_21 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1 + 2304 + d_lane]; - float bt_t_21 = btv_1[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1); - float e_dg_21 = approx_exp2(g_t_21 - gc_2); - float dq2_21 = __fmaf_rn(qo_1[2], e_off_21, qdg_1[2] * e_dg_21); - float dk2_21 = __fmaf_rn(ko_1[2], e_off_21, kdg_1[2] * e_dg_21); - dbp_1[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + dkt_2[2]; - float dg_o_21 = __fmaf_rn(q_t_1_21, dq2_21, (dk2b_21 - dkt_2[2]) * k_t_21) + dgf_21; - s_run_1[0] = s_run_1[0] + dg_o_21; +#if __CUDA_ARCH__ == 1030 + int t_13 = 16 + 8 * th + 2; + float r_292[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_13 = r_292[0]; + unsigned int r_293[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_13 = __uint_as_float(r_293[0] << 16); + unsigned int r_294[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_13 = __uint_as_float(r_294[0] << 16); + float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); + float bt_t_13 = btv_0[2]; + float e_off_13 = approx_exp2(g_t_13 - gn_1); + float e_dg_13 = approx_exp2(g_t_13 - gc_2); + float dq2_13 = __fmaf_rn(qo_0[2], e_off_13, qdg_0[2] * e_dg_13); + float dk2_13 = __fmaf_rn(ko_0[2], e_off_13, kdg_0[2] * e_dg_13); + dbp_1[2] = dk2_13 * k_t_13; + float dk2b_13 = dk2_13 * bt_t_13; + float qe_12 = q_t_1_13 * e_dg_13; + float kp_24 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_13) * kdg_0[2]); + float dqf_13 = pool_f32[ew_1 + 256 + d_lane]; + float dkf_13 = pool_f32[ew_1 + 1280 + d_lane]; + float dgf_13 = pool_f32[ew_1 + 2304 + d_lane]; + float dq_o_13 = dq2_13 + dqf_13; + float dk_o_13 = dk2b_13 + dkf_13 + dkt_2[2]; + float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_2[2], k_t_13, cr_12)) + dgf_13; + s_run_1[0] = s_run_1[0] + dg_o_13; +#else + int t_5 = 16 + 8 * th + 2; + float r_271[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_5 = r_271[0]; + unsigned int r_272[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_5 = __uint_as_float(r_272[0] << 16); + unsigned int r_273[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_5 = __uint_as_float(r_273[0] << 16); + float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); + float bt_t_5 = btv_2[2]; + float e_off_5 = approx_exp2(g_t_5 - gn_1); + float e_dg_5 = approx_exp2(g_t_5 - gc_2); + float dq2_5 = __fmaf_rn(qo_2[2], e_off_5, qdg_2[2] * e_dg_5); + float dk2_5 = __fmaf_rn(ko_2[2], e_off_5, kdg_2[2] * e_dg_5); + dbp_1[2] = dk2_5 * k_t_5; + float dk2b_5 = dk2_5 * bt_t_5; + float qe_4 = q_t_1_5 * e_dg_5; + float kp_16 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg_2[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_5) * kdg_2[2]); + float dqf_5 = pool_f32[ew_1 + 256 + d_lane]; + float dkf_5 = pool_f32[ew_1 + 1280 + d_lane]; + float dgf_5 = pool_f32[ew_1 + 2304 + d_lane]; + float dq_o_5 = dq2_5 + dqf_5; + float dk_o_5 = dk2b_5 + dkf_5 + dkt_2[2]; + float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_2[2], k_t_5, cr_4)) + dgf_5; + s_run_1[0] = s_run_1[0] + dg_o_5; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_21 = t_21 * qrs; - if (t_21 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_13 = t_13 * qrs; + if (t_13 < clen) { +#else + int qoff_5 = t_5 * qrs; + if (t_5 < clen) { +#endif { - auto _gst_406 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_406)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_21; +#if __CUDA_ARCH__ == 1030 + auto _gst_424 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_424)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_424) + (0)) = (__nv_bfloat16)dq_o_13; +#else + auto _gst_410 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_410)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_5; +#endif } { - auto _gst_407 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_407)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_21; +#if __CUDA_ARCH__ == 1030 + auto _gst_425 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_425)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_425) + (0)) = (__nv_bfloat16)dk_o_13; +#else + auto _gst_411 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_411)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_5; +#endif } } - int t_22 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1 + 2176 + d_lane]; - float bt_t_22 = btv_1[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1); - float e_dg_22 = approx_exp2(g_t_22 - gc_2); - float dq2_22 = __fmaf_rn(qo_1[1], e_off_22, qdg_1[1] * e_dg_22); - float dk2_22 = __fmaf_rn(ko_1[1], e_off_22, kdg_1[1] * e_dg_22); - dbp_1[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + dkt_2[1]; - float dg_o_22 = __fmaf_rn(q_t_1_22, dq2_22, (dk2b_22 - dkt_2[1]) * k_t_22) + dgf_22; - s_run_1[0] = s_run_1[0] + dg_o_22; +#if __CUDA_ARCH__ == 1030 + int t_14 = 16 + 8 * th + 1; + float r_295[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_14 = r_295[0]; + unsigned int r_296[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_14 = __uint_as_float(r_296[0] << 16); + unsigned int r_297[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_14 = __uint_as_float(r_297[0] << 16); + float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); + float bt_t_14 = btv_0[1]; + float e_off_14 = approx_exp2(g_t_14 - gn_1); + float e_dg_14 = approx_exp2(g_t_14 - gc_2); + float dq2_14 = __fmaf_rn(qo_0[1], e_off_14, qdg_0[1] * e_dg_14); + float dk2_14 = __fmaf_rn(ko_0[1], e_off_14, kdg_0[1] * e_dg_14); + dbp_1[1] = dk2_14 * k_t_14; + float dk2b_14 = dk2_14 * bt_t_14; + float qe_13 = q_t_1_14 * e_dg_14; + float kp_25 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_14) * kdg_0[1]); + float dqf_14 = pool_f32[ew_1 + 128 + d_lane]; + float dkf_14 = pool_f32[ew_1 + 1152 + d_lane]; + float dgf_14 = pool_f32[ew_1 + 2176 + d_lane]; + float dq_o_14 = dq2_14 + dqf_14; + float dk_o_14 = dk2b_14 + dkf_14 + dkt_2[1]; + float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_2[1], k_t_14, cr_13)) + dgf_14; + s_run_1[0] = s_run_1[0] + dg_o_14; +#else + int t_6 = 16 + 8 * th + 1; + float r_274[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_6 = r_274[0]; + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_6 = __uint_as_float(r_275[0] << 16); + unsigned int r_276[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_6 = __uint_as_float(r_276[0] << 16); + float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); + float bt_t_6 = btv_2[1]; + float e_off_6 = approx_exp2(g_t_6 - gn_1); + float e_dg_6 = approx_exp2(g_t_6 - gc_2); + float dq2_6 = __fmaf_rn(qo_2[1], e_off_6, qdg_2[1] * e_dg_6); + float dk2_6 = __fmaf_rn(ko_2[1], e_off_6, kdg_2[1] * e_dg_6); + dbp_1[1] = dk2_6 * k_t_6; + float dk2b_6 = dk2_6 * bt_t_6; + float qe_5 = q_t_1_6 * e_dg_6; + float kp_17 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg_2[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_6) * kdg_2[1]); + float dqf_6 = pool_f32[ew_1 + 128 + d_lane]; + float dkf_6 = pool_f32[ew_1 + 1152 + d_lane]; + float dgf_6 = pool_f32[ew_1 + 2176 + d_lane]; + float dq_o_6 = dq2_6 + dqf_6; + float dk_o_6 = dk2b_6 + dkf_6 + dkt_2[1]; + float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_2[1], k_t_6, cr_5)) + dgf_6; + s_run_1[0] = s_run_1[0] + dg_o_6; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_22 = t_22 * qrs; - if (t_22 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_14 = t_14 * qrs; + if (t_14 < clen) { +#else + int qoff_6 = t_6 * qrs; + if (t_6 < clen) { +#endif { - auto _gst_408 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_408)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_22; +#if __CUDA_ARCH__ == 1030 + auto _gst_426 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_426)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_426) + (0)) = (__nv_bfloat16)dq_o_14; +#else + auto _gst_412 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_412)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_412) + (0)) = (__nv_bfloat16)dq_o_6; +#endif } { - auto _gst_409 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_409)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_22; +#if __CUDA_ARCH__ == 1030 + auto _gst_427 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_427)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_427) + (0)) = (__nv_bfloat16)dk_o_14; +#else + auto _gst_413 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_413)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_413) + (0)) = (__nv_bfloat16)dk_o_6; +#endif } } - int t_23 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1 + d_lane]; - float dkf_23 = pool_f32[ew_1 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1 + 2048 + d_lane]; - float bt_t_23 = btv_1[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1); - float e_dg_23 = approx_exp2(g_t_23 - gc_2); - float dq2_23 = __fmaf_rn(qo_1[0], e_off_23, qdg_1[0] * e_dg_23); - float dk2_23 = __fmaf_rn(ko_1[0], e_off_23, kdg_1[0] * e_dg_23); - dbp_1[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + dkt_2[0]; - float dg_o_23 = __fmaf_rn(q_t_1_23, dq2_23, (dk2b_23 - dkt_2[0]) * k_t_23) + dgf_23; - s_run_1[0] = s_run_1[0] + dg_o_23; +#if __CUDA_ARCH__ == 1030 + int t_15 = 16 + 8 * th; + float r_298[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_15 = r_298[0]; + unsigned int r_299[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_15 = __uint_as_float(r_299[0] << 16); + unsigned int r_300[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_15 = __uint_as_float(r_300[0] << 16); + float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); + float bt_t_15 = btv_0[0]; + float e_off_15 = approx_exp2(g_t_15 - gn_1); + float e_dg_15 = approx_exp2(g_t_15 - gc_2); + float dq2_15 = __fmaf_rn(qo_0[0], e_off_15, qdg_0[0] * e_dg_15); + float dk2_15 = __fmaf_rn(ko_0[0], e_off_15, kdg_0[0] * e_dg_15); + dbp_1[0] = dk2_15 * k_t_15; + float dk2b_15 = dk2_15 * bt_t_15; + float qe_14 = q_t_1_15 * e_dg_15; + float kp_26 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_15) * kdg_0[0]); + float dqf_15 = pool_f32[ew_1 + d_lane]; + float dkf_15 = pool_f32[ew_1 + 1024 + d_lane]; + float dgf_15 = pool_f32[ew_1 + 2048 + d_lane]; + float dq_o_15 = dq2_15 + dqf_15; + float dk_o_15 = dk2b_15 + dkf_15 + dkt_2[0]; + float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_2[0], k_t_15, cr_14)) + dgf_15; + s_run_1[0] = s_run_1[0] + dg_o_15; +#else + int t_7 = 16 + 8 * th; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_7 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_7 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_7 = __uint_as_float(r_279[0] << 16); + float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); + float bt_t_7 = btv_2[0]; + float e_off_7 = approx_exp2(g_t_7 - gn_1); + float e_dg_7 = approx_exp2(g_t_7 - gc_2); + float dq2_7 = __fmaf_rn(qo_2[0], e_off_7, qdg_2[0] * e_dg_7); + float dk2_7 = __fmaf_rn(ko_2[0], e_off_7, kdg_2[0] * e_dg_7); + dbp_1[0] = dk2_7 * k_t_7; + float dk2b_7 = dk2_7 * bt_t_7; + float qe_6 = q_t_1_7 * e_dg_7; + float kp_18 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg_2[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_7) * kdg_2[0]); + float dqf_7 = pool_f32[ew_1 + d_lane]; + float dkf_7 = pool_f32[ew_1 + 1024 + d_lane]; + float dgf_7 = pool_f32[ew_1 + 2048 + d_lane]; + float dq_o_7 = dq2_7 + dqf_7; + float dk_o_7 = dk2b_7 + dkf_7 + dkt_2[0]; + float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_2[0], k_t_7, cr_6)) + dgf_7; + s_run_1[0] = s_run_1[0] + dg_o_7; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_23 = t_23 * qrs; - if (t_23 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_15 = t_15 * qrs; + if (t_15 < clen) { +#else + int qoff_7 = t_7 * qrs; + if (t_7 < clen) { +#endif { - auto _gst_410 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_410)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_23; +#if __CUDA_ARCH__ == 1030 + auto _gst_428 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_428)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_428) + (0)) = (__nv_bfloat16)dq_o_15; +#else + auto _gst_414 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_414)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_414) + (0)) = (__nv_bfloat16)dq_o_7; +#endif } { - auto _gst_411 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_411)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_23; +#if __CUDA_ARCH__ == 1030 + auto _gst_429 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_429)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_429) + (0)) = (__nv_bfloat16)dk_o_15; +#else + auto _gst_415 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_415)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_415) + (0)) = (__nv_bfloat16)dk_o_7; +#endif } } float tot_1 = s_run_1[0]; @@ -4860,39 +5859,83 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_412 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_412; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_430 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_430; +#else + float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_416; +#endif float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_413; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_431 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_431; +#else + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_417; +#endif float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_414 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_414; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_432 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_432; +#else + float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_418; +#endif float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_415; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_433 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_433; +#else + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_419; +#endif float v2_17[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_17[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_416; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_434 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_17[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_434; +#else + float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_17[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_420; +#endif float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_17[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_417; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_435 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_17[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_435; +#else + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_17[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_421; +#endif float send_19 = ((b2_1 == 0) ? v2_17[1] : v2_17[0]); - float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_418; - float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_419; - float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_420; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_436 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_436; + float _shfl_xor_sync_437 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_437; + float _shfl_xor_sync_438 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_438; +#else + float _shfl_xor_sync_422 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_422; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_423; + float _shfl_xor_sync_424 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_424; +#endif asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_421 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_421), "f"(tot_1) : "memory"); +#if __CUDA_ARCH__ == 1030 + uint32_t _addr_439 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_439), "f"(tot_1) : "memory"); +#else + uint32_t _addr_425 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_425), "f"(tot_1) : "memory"); +#endif } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4924,132 +5967,280 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_422 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_440 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_426 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_422), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_425; - asm("abs.f32 %0, %1;" : "=f"(_abs_425) : "f"(rp_z_7.x)); - float _neg_424; - asm("neg.f32 %0, %1;" : "=f"(_neg_424) : "f"(_abs_425)); - float _exp_423 = __nv_expf(_neg_424); - float _abs_428; - asm("abs.f32 %0, %1;" : "=f"(_abs_428) : "f"(rp_z_7.y)); - float _neg_427; - asm("neg.f32 %0, %1;" : "=f"(_neg_427) : "f"(_abs_428)); - float _exp_426 = __nv_expf(_neg_427); - float2 rp_e_7 = make_float2(_exp_423, _exp_426); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_440), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_443; + asm("abs.f32 %0, %1;" : "=f"(_abs_443) : "f"(rp_z_7.x)); + float _neg_442; + asm("neg.f32 %0, %1;" : "=f"(_neg_442) : "f"(_abs_443)); + float _exp_441 = __nv_expf(_neg_442); + float _abs_446; + asm("abs.f32 %0, %1;" : "=f"(_abs_446) : "f"(rp_z_7.y)); + float _neg_445; + asm("neg.f32 %0, %1;" : "=f"(_neg_445) : "f"(_abs_446)); + float _exp_444 = __nv_expf(_neg_445); + float2 rp_e_7 = make_float2(_exp_441, _exp_444); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_426), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_429; + asm("abs.f32 %0, %1;" : "=f"(_abs_429) : "f"(rp_z_7.x)); + float _neg_428; + asm("neg.f32 %0, %1;" : "=f"(_neg_428) : "f"(_abs_429)); + float _exp_427 = __nv_expf(_neg_428); + float _abs_432; + asm("abs.f32 %0, %1;" : "=f"(_abs_432) : "f"(rp_z_7.y)); + float _neg_431; + asm("neg.f32 %0, %1;" : "=f"(_neg_431) : "f"(_abs_432)); + float _exp_430 = __nv_expf(_neg_431); + float2 rp_e_7 = make_float2(_exp_427, _exp_430); +#endif float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_429 = approx_rcp(rp_den_7.x); - float _rcp_430 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_431 = make_float2(_rcp_429, _rcp_430); +#if __CUDA_ARCH__ == 1030 + float _rcp_447 = approx_rcp(rp_den_7.x); + float _rcp_448 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_449 = make_float2(_rcp_447, _rcp_448); +#else + float _rcp_433 = approx_rcp(rp_den_7.x); + float _rcp_434 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_435 = make_float2(_rcp_433, _rcp_434); +#endif float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_431)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_449)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_435)); +#endif float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_432 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_450 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_436 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_432)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_450)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_436)); +#endif u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_433 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_451 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_437 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_433), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_436; - asm("abs.f32 %0, %1;" : "=f"(_abs_436) : "f"(rp_z_8.x)); - float _neg_435; - asm("neg.f32 %0, %1;" : "=f"(_neg_435) : "f"(_abs_436)); - float _exp_434 = __nv_expf(_neg_435); - float _abs_439; - asm("abs.f32 %0, %1;" : "=f"(_abs_439) : "f"(rp_z_8.y)); - float _neg_438; - asm("neg.f32 %0, %1;" : "=f"(_neg_438) : "f"(_abs_439)); - float _exp_437 = __nv_expf(_neg_438); - float2 rp_e_8 = make_float2(_exp_434, _exp_437); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_451), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_454; + asm("abs.f32 %0, %1;" : "=f"(_abs_454) : "f"(rp_z_8.x)); + float _neg_453; + asm("neg.f32 %0, %1;" : "=f"(_neg_453) : "f"(_abs_454)); + float _exp_452 = __nv_expf(_neg_453); + float _abs_457; + asm("abs.f32 %0, %1;" : "=f"(_abs_457) : "f"(rp_z_8.y)); + float _neg_456; + asm("neg.f32 %0, %1;" : "=f"(_neg_456) : "f"(_abs_457)); + float _exp_455 = __nv_expf(_neg_456); + float2 rp_e_8 = make_float2(_exp_452, _exp_455); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_437), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_440; + asm("abs.f32 %0, %1;" : "=f"(_abs_440) : "f"(rp_z_8.x)); + float _neg_439; + asm("neg.f32 %0, %1;" : "=f"(_neg_439) : "f"(_abs_440)); + float _exp_438 = __nv_expf(_neg_439); + float _abs_443; + asm("abs.f32 %0, %1;" : "=f"(_abs_443) : "f"(rp_z_8.y)); + float _neg_442; + asm("neg.f32 %0, %1;" : "=f"(_neg_442) : "f"(_abs_443)); + float _exp_441 = __nv_expf(_neg_442); + float2 rp_e_8 = make_float2(_exp_438, _exp_441); +#endif float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_440 = approx_rcp(rp_den_8.x); - float _rcp_441 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_442 = make_float2(_rcp_440, _rcp_441); +#if __CUDA_ARCH__ == 1030 + float _rcp_458 = approx_rcp(rp_den_8.x); + float _rcp_459 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_460 = make_float2(_rcp_458, _rcp_459); +#else + float _rcp_444 = approx_rcp(rp_den_8.x); + float _rcp_445 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_446 = make_float2(_rcp_444, _rcp_445); +#endif float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_442)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_460)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_446)); +#endif float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_443 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_461 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_447 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_443)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_461)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_447)); +#endif u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_444 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_462 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_448 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_444), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_447; - asm("abs.f32 %0, %1;" : "=f"(_abs_447) : "f"(rp_z_9.x)); - float _neg_446; - asm("neg.f32 %0, %1;" : "=f"(_neg_446) : "f"(_abs_447)); - float _exp_445 = __nv_expf(_neg_446); - float _abs_450; - asm("abs.f32 %0, %1;" : "=f"(_abs_450) : "f"(rp_z_9.y)); - float _neg_449; - asm("neg.f32 %0, %1;" : "=f"(_neg_449) : "f"(_abs_450)); - float _exp_448 = __nv_expf(_neg_449); - float2 rp_e_9 = make_float2(_exp_445, _exp_448); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_462), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_465; + asm("abs.f32 %0, %1;" : "=f"(_abs_465) : "f"(rp_z_9.x)); + float _neg_464; + asm("neg.f32 %0, %1;" : "=f"(_neg_464) : "f"(_abs_465)); + float _exp_463 = __nv_expf(_neg_464); + float _abs_468; + asm("abs.f32 %0, %1;" : "=f"(_abs_468) : "f"(rp_z_9.y)); + float _neg_467; + asm("neg.f32 %0, %1;" : "=f"(_neg_467) : "f"(_abs_468)); + float _exp_466 = __nv_expf(_neg_467); + float2 rp_e_9 = make_float2(_exp_463, _exp_466); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_448), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_451; + asm("abs.f32 %0, %1;" : "=f"(_abs_451) : "f"(rp_z_9.x)); + float _neg_450; + asm("neg.f32 %0, %1;" : "=f"(_neg_450) : "f"(_abs_451)); + float _exp_449 = __nv_expf(_neg_450); + float _abs_454; + asm("abs.f32 %0, %1;" : "=f"(_abs_454) : "f"(rp_z_9.y)); + float _neg_453; + asm("neg.f32 %0, %1;" : "=f"(_neg_453) : "f"(_abs_454)); + float _exp_452 = __nv_expf(_neg_453); + float2 rp_e_9 = make_float2(_exp_449, _exp_452); +#endif float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_451 = approx_rcp(rp_den_9.x); - float _rcp_452 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_453 = make_float2(_rcp_451, _rcp_452); +#if __CUDA_ARCH__ == 1030 + float _rcp_469 = approx_rcp(rp_den_9.x); + float _rcp_470 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_471 = make_float2(_rcp_469, _rcp_470); +#else + float _rcp_455 = approx_rcp(rp_den_9.x); + float _rcp_456 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_457 = make_float2(_rcp_455, _rcp_456); +#endif float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_453)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_471)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_457)); +#endif float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_454 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_472 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_458 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_454)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_472)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); +#endif u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_455 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_473 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_459 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_455), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_458; - asm("abs.f32 %0, %1;" : "=f"(_abs_458) : "f"(rp_z_10.x)); - float _neg_457; - asm("neg.f32 %0, %1;" : "=f"(_neg_457) : "f"(_abs_458)); - float _exp_456 = __nv_expf(_neg_457); - float _abs_461; - asm("abs.f32 %0, %1;" : "=f"(_abs_461) : "f"(rp_z_10.y)); - float _neg_460; - asm("neg.f32 %0, %1;" : "=f"(_neg_460) : "f"(_abs_461)); - float _exp_459 = __nv_expf(_neg_460); - float2 rp_e_10 = make_float2(_exp_456, _exp_459); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_473), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_476; + asm("abs.f32 %0, %1;" : "=f"(_abs_476) : "f"(rp_z_10.x)); + float _neg_475; + asm("neg.f32 %0, %1;" : "=f"(_neg_475) : "f"(_abs_476)); + float _exp_474 = __nv_expf(_neg_475); + float _abs_479; + asm("abs.f32 %0, %1;" : "=f"(_abs_479) : "f"(rp_z_10.y)); + float _neg_478; + asm("neg.f32 %0, %1;" : "=f"(_neg_478) : "f"(_abs_479)); + float _exp_477 = __nv_expf(_neg_478); + float2 rp_e_10 = make_float2(_exp_474, _exp_477); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_459), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_462; + asm("abs.f32 %0, %1;" : "=f"(_abs_462) : "f"(rp_z_10.x)); + float _neg_461; + asm("neg.f32 %0, %1;" : "=f"(_neg_461) : "f"(_abs_462)); + float _exp_460 = __nv_expf(_neg_461); + float _abs_465; + asm("abs.f32 %0, %1;" : "=f"(_abs_465) : "f"(rp_z_10.y)); + float _neg_464; + asm("neg.f32 %0, %1;" : "=f"(_neg_464) : "f"(_abs_465)); + float _exp_463 = __nv_expf(_neg_464); + float2 rp_e_10 = make_float2(_exp_460, _exp_463); +#endif float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_462 = approx_rcp(rp_den_10.x); - float _rcp_463 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_464 = make_float2(_rcp_462, _rcp_463); +#if __CUDA_ARCH__ == 1030 + float _rcp_480 = approx_rcp(rp_den_10.x); + float _rcp_481 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_482 = make_float2(_rcp_480, _rcp_481); +#else + float _rcp_466 = approx_rcp(rp_den_10.x); + float _rcp_467 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_468 = make_float2(_rcp_466, _rcp_467); +#endif float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_464)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_482)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_468)); +#endif float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_465 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_483 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_469 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_465)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_483)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); +#endif u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -5064,54 +6255,102 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_466; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_466) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_466; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_484; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_484) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_484; +#else + uint32_t _prmt_470; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_470; +#endif int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_467 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_467)); - *(reinterpret_cast(_gst_467) + (0)) = pq_8[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_485 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_485)); + *(reinterpret_cast(_gst_485) + (0)) = pq_8[0]; +#else + auto _gst_471 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_471)); + *(reinterpret_cast(_gst_471) + (0)) = pq_8[0]; +#endif } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_468; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_468) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_468; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_486; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_486) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_486; +#else + uint32_t _prmt_472; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_472; +#endif int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_469 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_469)); - *(reinterpret_cast(_gst_469) + (0)) = pq_9[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_487 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_487)); + *(reinterpret_cast(_gst_487) + (0)) = pq_9[0]; +#else + auto _gst_473 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_473)); + *(reinterpret_cast(_gst_473) + (0)) = pq_9[0]; +#endif } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_470; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_470; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_488; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_488) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_488; +#else + uint32_t _prmt_474; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_474) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_474; +#endif int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_471 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_471)); - *(reinterpret_cast(_gst_471) + (0)) = pq_10[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_489 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_489)); + *(reinterpret_cast(_gst_489) + (0)) = pq_10[0]; +#else + auto _gst_475 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_475)); + *(reinterpret_cast(_gst_475) + (0)) = pq_10[0]; +#endif } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_472; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_472; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_490; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_490) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_490; +#else + uint32_t _prmt_476; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_476) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_476; +#endif int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_473 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_473)); - *(reinterpret_cast(_gst_473) + (0)) = pq_11[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_491 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_491)); + *(reinterpret_cast(_gst_491) + (0)) = pq_11[0]; +#else + auto _gst_477 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_477)); + *(reinterpret_cast(_gst_477) + (0)) = pq_11[0]; +#endif } } int t_a_16 = 16 + 8 * th; @@ -5176,47 +6415,99 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_474; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_474) : "f"(s_a_1), "f"(nm_t_3)); +#if __CUDA_ARCH__ == 1030 + float _sub_492; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_492) : "f"(s_a_1), "f"(nm_t_3)); +#else + float _sub_478; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_478) : "f"(s_a_1), "f"(nm_t_3)); +#endif float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_474), "f"(pa_1)); - float _sub_475; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_475) : "f"(pa_1), "f"(nm_t_3)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_492), "f"(pa_1)); + float _sub_493; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_493) : "f"(pa_1), "f"(nm_t_3)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_478), "f"(pa_1)); + float _sub_479; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_479) : "f"(pa_1), "f"(nm_t_3)); +#endif float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_475), "f"(s_a_1)); - float _abs_477; - asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(s_a_1)); - float _abs_478; - asm("abs.f32 %0, %1;" : "=f"(_abs_478) : "f"(pa_1)); - uint32_t _setp_476_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_476_bits) : "f"(_abs_477), "f"(_abs_478)); - bool _setp_476 = _setp_476_bits != 0; - float nm_err_3 = ((_setp_476) ? nm_es_3 : nm_ex_3); - float _add_479; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_479) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_479; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_493), "f"(s_a_1)); + float _abs_495; + asm("abs.f32 %0, %1;" : "=f"(_abs_495) : "f"(s_a_1)); + float _abs_496; + asm("abs.f32 %0, %1;" : "=f"(_abs_496) : "f"(pa_1)); + uint32_t _setp_494_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_494_bits) : "f"(_abs_495), "f"(_abs_496)); + bool _setp_494 = _setp_494_bits != 0; + float nm_err_3 = ((_setp_494) ? nm_es_3 : nm_ex_3); + float _add_497; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_497) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_497; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_479), "f"(s_a_1)); + float _abs_481; + asm("abs.f32 %0, %1;" : "=f"(_abs_481) : "f"(s_a_1)); + float _abs_482; + asm("abs.f32 %0, %1;" : "=f"(_abs_482) : "f"(pa_1)); + uint32_t _setp_480_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_480_bits) : "f"(_abs_481), "f"(_abs_482)); + bool _setp_480 = _setp_480_bits != 0; + float nm_err_3 = ((_setp_480) ? nm_es_3 : nm_ex_3); + float _add_483; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_483) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_483; +#endif s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_480; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_480) : "f"(s_b_1), "f"(nm_t_4)); +#if __CUDA_ARCH__ == 1030 + float _sub_498; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_498) : "f"(s_b_1), "f"(nm_t_4)); +#else + float _sub_484; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_484) : "f"(s_b_1), "f"(nm_t_4)); +#endif float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_480), "f"(pb_1)); - float _sub_481; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_481) : "f"(pb_1), "f"(nm_t_4)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_498), "f"(pb_1)); + float _sub_499; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_499) : "f"(pb_1), "f"(nm_t_4)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_484), "f"(pb_1)); + float _sub_485; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_485) : "f"(pb_1), "f"(nm_t_4)); +#endif float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_481), "f"(s_b_1)); - float _abs_483; - asm("abs.f32 %0, %1;" : "=f"(_abs_483) : "f"(s_b_1)); - float _abs_484; - asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(pb_1)); - uint32_t _setp_482_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_482_bits) : "f"(_abs_483), "f"(_abs_484)); - bool _setp_482 = _setp_482_bits != 0; - float nm_err_4 = ((_setp_482) ? nm_es_4 : nm_ex_4); - float _add_485; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_485) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_485; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_499), "f"(s_b_1)); + float _abs_501; + asm("abs.f32 %0, %1;" : "=f"(_abs_501) : "f"(s_b_1)); + float _abs_502; + asm("abs.f32 %0, %1;" : "=f"(_abs_502) : "f"(pb_1)); + uint32_t _setp_500_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_500_bits) : "f"(_abs_501), "f"(_abs_502)); + bool _setp_500 = _setp_500_bits != 0; + float nm_err_4 = ((_setp_500) ? nm_es_4 : nm_ex_4); + float _add_503; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_503) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_503; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_485), "f"(s_b_1)); + float _abs_487; + asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(s_b_1)); + float _abs_488; + asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(pb_1)); + uint32_t _setp_486_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_486_bits) : "f"(_abs_487), "f"(_abs_488)); + bool _setp_486 = _setp_486_bits != 0; + float nm_err_4 = ((_setp_486) ? nm_es_4 : nm_ex_4); + float _add_489; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_489) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_489; +#endif s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5224,8 +6515,13 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_486 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_486), "f"(off_acc[0]) : "memory"); +#if __CUDA_ARCH__ == 1030 + uint32_t _addr_504 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_504), "f"(off_acc[0]) : "memory"); +#else + uint32_t _addr_490 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_490), "f"(off_acc[0]) : "memory"); +#endif } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5249,9 +6545,15 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_2 = r_322[0]; +#if __CUDA_ARCH__ == 1030 + float r_301[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_2 = r_301[0]; +#else + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_2 = r_280[0]; +#endif float gc_3 = gn_2 - 21.640425f; int eb_2 = 98304 + th * -32768; int ew_2 = eb_2 >> 2; @@ -5260,364 +6562,823 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int gb8_2 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_2 * 128); int kb8_2 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_2 * 128); float dbp_2[8]; - float qdg_2[8]; - tmem_ld_x8(&qdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float kdg_2[8]; - tmem_ld_x8(&kdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); +#if __CUDA_ARCH__ == 1030 + float qdg_1_1[8]; + tmem_ld_x8(&qdg_1_1[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); + float kdg_1_1[8]; + tmem_ld_x8(&kdg_1_1[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); +#else + float qdg_0[8]; + tmem_ld_x8(&qdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); + float kdg_0[8]; + tmem_ld_x8(&kdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); +#endif float dkt_3[8]; tmem_ld_x8(&dkt_3[0], taddr + (unsigned int)row_base + (unsigned int)(8 * th)); - float btv_2[8]; +#if __CUDA_ARCH__ == 1030 + float btv_1_1[8]; +#else + float btv_0[8]; +#endif asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_2[0])), "=r"(*reinterpret_cast(&btv_2[(0) + 1])), "=r"(*reinterpret_cast(&btv_2[(0) + 2])), "=r"(*reinterpret_cast(&btv_2[(0) + 3])) +#if __CUDA_ARCH__ == 1030 + : "=r"(*reinterpret_cast(&btv_1_1[0])), "=r"(*reinterpret_cast(&btv_1_1[(0) + 1])), "=r"(*reinterpret_cast(&btv_1_1[(0) + 2])), "=r"(*reinterpret_cast(&btv_1_1[(0) + 3])) +#else + : "=r"(*reinterpret_cast(&btv_0[0])), "=r"(*reinterpret_cast(&btv_0[(0) + 1])), "=r"(*reinterpret_cast(&btv_0[(0) + 2])), "=r"(*reinterpret_cast(&btv_0[(0) + 3])) +#endif : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_2[4])), "=r"(*reinterpret_cast(&btv_2[(4) + 1])), "=r"(*reinterpret_cast(&btv_2[(4) + 2])), "=r"(*reinterpret_cast(&btv_2[(4) + 3])) +#if __CUDA_ARCH__ == 1030 + : "=r"(*reinterpret_cast(&btv_1_1[4])), "=r"(*reinterpret_cast(&btv_1_1[(4) + 1])), "=r"(*reinterpret_cast(&btv_1_1[(4) + 2])), "=r"(*reinterpret_cast(&btv_1_1[(4) + 3])) +#else + : "=r"(*reinterpret_cast(&btv_0[4])), "=r"(*reinterpret_cast(&btv_0[(4) + 1])), "=r"(*reinterpret_cast(&btv_0[(4) + 2])), "=r"(*reinterpret_cast(&btv_0[(4) + 3])) +#endif : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th) + 16))); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_2[1]; s_run_2[0] = 0.0f; - int t_24 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_24 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_24 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_24 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_24 >= clen) ? 0.0f : q_t_24); - float dqf_24 = pool_f32[ew_2 + 896 + d_lane]; - float dkf_24 = pool_f32[ew_2 + 1920 + d_lane]; - float dgf_24 = pool_f32[ew_2 + 2944 + d_lane]; - float bt_t_24 = btv_2[7]; - float e_off_24 = approx_exp2(g_t_24 - gn_2); - float e_dg_24 = approx_exp2(g_t_24 - gc_3); - float dq2_24 = qdg_2[7] * e_dg_24; - float dk2_24 = kdg_2[7] * e_dg_24; - dbp_2[7] = dk2_24 * k_t_24; - float dk2b_24 = dk2_24 * bt_t_24; - float dq_o_24 = dq2_24 + dqf_24; - float dk_o_24 = dk2b_24 + dkf_24 + dkt_3[7]; - float dg_o_24 = __fmaf_rn(q_t_1_24, dq2_24, (dk2b_24 - dkt_3[7]) * k_t_24) + dgf_24; - s_run_2[0] = s_run_2[0] + dg_o_24; +#if __CUDA_ARCH__ == 1030 + int t_16 = 8 * th + 7; + float r_302[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_16 = r_302[0]; + unsigned int r_303[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_16 = __uint_as_float(r_303[0] << 16); + unsigned int r_304[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_16 = __uint_as_float(r_304[0] << 16); + float q_t_1_16 = ((t_16 >= clen) ? 0.0f : q_t_16); + float bt_t_16 = btv_1_1[7]; + float e_off_16 = approx_exp2(g_t_16 - gn_2); + float e_dg_16 = approx_exp2(g_t_16 - gc_3); + float dq2_16 = qdg_1_1[7] * e_dg_16; + float dk2_16 = kdg_1_1[7] * e_dg_16; + dbp_2[7] = dk2_16 * k_t_16; + float dk2b_16 = dk2_16 * bt_t_16; + float qe_15 = q_t_1_16 * e_dg_16; + float kp_27 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1_1[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_16) * kdg_1_1[7]); + float dqf_16 = pool_f32[ew_2 + 896 + d_lane]; + float dkf_16 = pool_f32[ew_2 + 1920 + d_lane]; + float dgf_16 = pool_f32[ew_2 + 2944 + d_lane]; + float dq_o_16 = dq2_16 + dqf_16; + float dk_o_16 = dk2b_16 + dkf_16 + dkt_3[7]; + float dg_o_16 = __fmaf_rn(q_t_1_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_3[7], k_t_16, cr_15)) + dgf_16; + s_run_2[0] = s_run_2[0] + dg_o_16; +#else + int t_8 = 8 * th + 7; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_8 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_8 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_8 = __uint_as_float(r_283[0] << 16); + float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); + float bt_t_8 = btv_0[7]; + float e_off_8 = approx_exp2(g_t_8 - gn_2); + float e_dg_8 = approx_exp2(g_t_8 - gc_3); + float dq2_8 = qdg_0[7] * e_dg_8; + float dk2_8 = kdg_0[7] * e_dg_8; + dbp_2[7] = dk2_8 * k_t_8; + float dk2b_8 = dk2_8 * bt_t_8; + float qe_7 = q_t_1_8 * e_dg_8; + float kp_19 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_8) * kdg_0[7]); + float dqf_8 = pool_f32[ew_2 + 896 + d_lane]; + float dkf_8 = pool_f32[ew_2 + 1920 + d_lane]; + float dgf_8 = pool_f32[ew_2 + 2944 + d_lane]; + float dq_o_8 = dq2_8 + dqf_8; + float dk_o_8 = dk2b_8 + dkf_8 + dkt_3[7]; + float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_3[7], k_t_8, cr_7)) + dgf_8; + s_run_2[0] = s_run_2[0] + dg_o_8; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_24 = t_24 * qrs; - if (t_24 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_16 = t_16 * qrs; + if (t_16 < clen) { +#else + int qoff_8 = t_8 * qrs; + if (t_8 < clen) { +#endif { - auto _gst_487 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_487)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_487) + (0)) = (__nv_bfloat16)dq_o_24; +#if __CUDA_ARCH__ == 1030 + auto _gst_505 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_505)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_505) + (0)) = (__nv_bfloat16)dq_o_16; +#else + auto _gst_491 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_491)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_8; +#endif } { - auto _gst_488 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_488)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_488) + (0)) = (__nv_bfloat16)dk_o_24; +#if __CUDA_ARCH__ == 1030 + auto _gst_506 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_506)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_506) + (0)) = (__nv_bfloat16)dk_o_16; +#else + auto _gst_492 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_492)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_8; +#endif } } - int t_25 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_25 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_25 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_25 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_25 >= clen) ? 0.0f : q_t_25); - float dqf_25 = pool_f32[ew_2 + 768 + d_lane]; - float dkf_25 = pool_f32[ew_2 + 1792 + d_lane]; - float dgf_25 = pool_f32[ew_2 + 2816 + d_lane]; - float bt_t_25 = btv_2[6]; - float e_off_25 = approx_exp2(g_t_25 - gn_2); - float e_dg_25 = approx_exp2(g_t_25 - gc_3); - float dq2_25 = qdg_2[6] * e_dg_25; - float dk2_25 = kdg_2[6] * e_dg_25; - dbp_2[6] = dk2_25 * k_t_25; - float dk2b_25 = dk2_25 * bt_t_25; - float dq_o_25 = dq2_25 + dqf_25; - float dk_o_25 = dk2b_25 + dkf_25 + dkt_3[6]; - float dg_o_25 = __fmaf_rn(q_t_1_25, dq2_25, (dk2b_25 - dkt_3[6]) * k_t_25) + dgf_25; - s_run_2[0] = s_run_2[0] + dg_o_25; +#if __CUDA_ARCH__ == 1030 + int t_17 = 8 * th + 6; + float r_305[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_17 = r_305[0]; + unsigned int r_306[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_17 = __uint_as_float(r_306[0] << 16); + unsigned int r_307[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_17 = __uint_as_float(r_307[0] << 16); + float q_t_1_17 = ((t_17 >= clen) ? 0.0f : q_t_17); + float bt_t_17 = btv_1_1[6]; + float e_off_17 = approx_exp2(g_t_17 - gn_2); + float e_dg_17 = approx_exp2(g_t_17 - gc_3); + float dq2_17 = qdg_1_1[6] * e_dg_17; + float dk2_17 = kdg_1_1[6] * e_dg_17; + dbp_2[6] = dk2_17 * k_t_17; + float dk2b_17 = dk2_17 * bt_t_17; + float qe_16 = q_t_1_17 * e_dg_17; + float kp_28 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1_1[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_17) * kdg_1_1[6]); + float dqf_17 = pool_f32[ew_2 + 768 + d_lane]; + float dkf_17 = pool_f32[ew_2 + 1792 + d_lane]; + float dgf_17 = pool_f32[ew_2 + 2816 + d_lane]; + float dq_o_17 = dq2_17 + dqf_17; + float dk_o_17 = dk2b_17 + dkf_17 + dkt_3[6]; + float dg_o_17 = __fmaf_rn(q_t_1_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_3[6], k_t_17, cr_16)) + dgf_17; + s_run_2[0] = s_run_2[0] + dg_o_17; +#else + int t_9 = 8 * th + 6; + float r_284[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_9 = r_284[0]; + unsigned int r_285[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_9 = __uint_as_float(r_285[0] << 16); + unsigned int r_286[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_9 = __uint_as_float(r_286[0] << 16); + float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); + float bt_t_9 = btv_0[6]; + float e_off_9 = approx_exp2(g_t_9 - gn_2); + float e_dg_9 = approx_exp2(g_t_9 - gc_3); + float dq2_9 = qdg_0[6] * e_dg_9; + float dk2_9 = kdg_0[6] * e_dg_9; + dbp_2[6] = dk2_9 * k_t_9; + float dk2b_9 = dk2_9 * bt_t_9; + float qe_8 = q_t_1_9 * e_dg_9; + float kp_20 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_9) * kdg_0[6]); + float dqf_9 = pool_f32[ew_2 + 768 + d_lane]; + float dkf_9 = pool_f32[ew_2 + 1792 + d_lane]; + float dgf_9 = pool_f32[ew_2 + 2816 + d_lane]; + float dq_o_9 = dq2_9 + dqf_9; + float dk_o_9 = dk2b_9 + dkf_9 + dkt_3[6]; + float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_3[6], k_t_9, cr_8)) + dgf_9; + s_run_2[0] = s_run_2[0] + dg_o_9; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_25 = t_25 * qrs; - if (t_25 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_17 = t_17 * qrs; + if (t_17 < clen) { +#else + int qoff_9 = t_9 * qrs; + if (t_9 < clen) { +#endif { - auto _gst_489 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_489)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_489) + (0)) = (__nv_bfloat16)dq_o_25; +#if __CUDA_ARCH__ == 1030 + auto _gst_507 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_507)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_507) + (0)) = (__nv_bfloat16)dq_o_17; +#else + auto _gst_493 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_493)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_9; +#endif } { - auto _gst_490 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_490)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_490) + (0)) = (__nv_bfloat16)dk_o_25; +#if __CUDA_ARCH__ == 1030 + auto _gst_508 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_508)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_508) + (0)) = (__nv_bfloat16)dk_o_17; +#else + auto _gst_494 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_494)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_9; +#endif } } - int t_26 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_26 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_26 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_26 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_26 >= clen) ? 0.0f : q_t_26); - float dqf_26 = pool_f32[ew_2 + 640 + d_lane]; - float dkf_26 = pool_f32[ew_2 + 1664 + d_lane]; - float dgf_26 = pool_f32[ew_2 + 2688 + d_lane]; - float bt_t_26 = btv_2[5]; - float e_off_26 = approx_exp2(g_t_26 - gn_2); - float e_dg_26 = approx_exp2(g_t_26 - gc_3); - float dq2_26 = qdg_2[5] * e_dg_26; - float dk2_26 = kdg_2[5] * e_dg_26; - dbp_2[5] = dk2_26 * k_t_26; - float dk2b_26 = dk2_26 * bt_t_26; - float dq_o_26 = dq2_26 + dqf_26; - float dk_o_26 = dk2b_26 + dkf_26 + dkt_3[5]; - float dg_o_26 = __fmaf_rn(q_t_1_26, dq2_26, (dk2b_26 - dkt_3[5]) * k_t_26) + dgf_26; - s_run_2[0] = s_run_2[0] + dg_o_26; +#if __CUDA_ARCH__ == 1030 + int t_18 = 8 * th + 5; + float r_308[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_18 = r_308[0]; + unsigned int r_309[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_18 = __uint_as_float(r_309[0] << 16); + unsigned int r_310[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_18 = __uint_as_float(r_310[0] << 16); + float q_t_1_18 = ((t_18 >= clen) ? 0.0f : q_t_18); + float bt_t_18 = btv_1_1[5]; + float e_off_18 = approx_exp2(g_t_18 - gn_2); + float e_dg_18 = approx_exp2(g_t_18 - gc_3); + float dq2_18 = qdg_1_1[5] * e_dg_18; + float dk2_18 = kdg_1_1[5] * e_dg_18; + dbp_2[5] = dk2_18 * k_t_18; + float dk2b_18 = dk2_18 * bt_t_18; + float qe_17 = q_t_1_18 * e_dg_18; + float kp_29 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1_1[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_18) * kdg_1_1[5]); + float dqf_18 = pool_f32[ew_2 + 640 + d_lane]; + float dkf_18 = pool_f32[ew_2 + 1664 + d_lane]; + float dgf_18 = pool_f32[ew_2 + 2688 + d_lane]; + float dq_o_18 = dq2_18 + dqf_18; + float dk_o_18 = dk2b_18 + dkf_18 + dkt_3[5]; + float dg_o_18 = __fmaf_rn(q_t_1_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_3[5], k_t_18, cr_17)) + dgf_18; + s_run_2[0] = s_run_2[0] + dg_o_18; +#else + int t_10 = 8 * th + 5; + float r_287[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_10 = r_287[0]; + unsigned int r_288[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_10 = __uint_as_float(r_288[0] << 16); + unsigned int r_289[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_10 = __uint_as_float(r_289[0] << 16); + float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); + float bt_t_10 = btv_0[5]; + float e_off_10 = approx_exp2(g_t_10 - gn_2); + float e_dg_10 = approx_exp2(g_t_10 - gc_3); + float dq2_10 = qdg_0[5] * e_dg_10; + float dk2_10 = kdg_0[5] * e_dg_10; + dbp_2[5] = dk2_10 * k_t_10; + float dk2b_10 = dk2_10 * bt_t_10; + float qe_9 = q_t_1_10 * e_dg_10; + float kp_21 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_10) * kdg_0[5]); + float dqf_10 = pool_f32[ew_2 + 640 + d_lane]; + float dkf_10 = pool_f32[ew_2 + 1664 + d_lane]; + float dgf_10 = pool_f32[ew_2 + 2688 + d_lane]; + float dq_o_10 = dq2_10 + dqf_10; + float dk_o_10 = dk2b_10 + dkf_10 + dkt_3[5]; + float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_3[5], k_t_10, cr_9)) + dgf_10; + s_run_2[0] = s_run_2[0] + dg_o_10; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_26 = t_26 * qrs; - if (t_26 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_18 = t_18 * qrs; + if (t_18 < clen) { +#else + int qoff_10 = t_10 * qrs; + if (t_10 < clen) { +#endif { - auto _gst_491 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_491)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_26; +#if __CUDA_ARCH__ == 1030 + auto _gst_509 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_509)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_509) + (0)) = (__nv_bfloat16)dq_o_18; +#else + auto _gst_495 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_495)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_10; +#endif } { - auto _gst_492 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_492)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_26; +#if __CUDA_ARCH__ == 1030 + auto _gst_510 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_510)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_510) + (0)) = (__nv_bfloat16)dk_o_18; +#else + auto _gst_496 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_496)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_10; +#endif } } - int t_27 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_27 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_27 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_27 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_27 >= clen) ? 0.0f : q_t_27); - float dqf_27 = pool_f32[ew_2 + 512 + d_lane]; - float dkf_27 = pool_f32[ew_2 + 1536 + d_lane]; - float dgf_27 = pool_f32[ew_2 + 2560 + d_lane]; - float bt_t_27 = btv_2[4]; - float e_off_27 = approx_exp2(g_t_27 - gn_2); - float e_dg_27 = approx_exp2(g_t_27 - gc_3); - float dq2_27 = qdg_2[4] * e_dg_27; - float dk2_27 = kdg_2[4] * e_dg_27; - dbp_2[4] = dk2_27 * k_t_27; - float dk2b_27 = dk2_27 * bt_t_27; - float dq_o_27 = dq2_27 + dqf_27; - float dk_o_27 = dk2b_27 + dkf_27 + dkt_3[4]; - float dg_o_27 = __fmaf_rn(q_t_1_27, dq2_27, (dk2b_27 - dkt_3[4]) * k_t_27) + dgf_27; - s_run_2[0] = s_run_2[0] + dg_o_27; +#if __CUDA_ARCH__ == 1030 + int t_19 = 8 * th + 4; + float r_311[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_19 = r_311[0]; + unsigned int r_312[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_19 = __uint_as_float(r_312[0] << 16); + unsigned int r_313[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_19 = __uint_as_float(r_313[0] << 16); + float q_t_1_19 = ((t_19 >= clen) ? 0.0f : q_t_19); + float bt_t_19 = btv_1_1[4]; + float e_off_19 = approx_exp2(g_t_19 - gn_2); + float e_dg_19 = approx_exp2(g_t_19 - gc_3); + float dq2_19 = qdg_1_1[4] * e_dg_19; + float dk2_19 = kdg_1_1[4] * e_dg_19; + dbp_2[4] = dk2_19 * k_t_19; + float dk2b_19 = dk2_19 * bt_t_19; + float qe_18 = q_t_1_19 * e_dg_19; + float kp_30 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1_1[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_19) * kdg_1_1[4]); + float dqf_19 = pool_f32[ew_2 + 512 + d_lane]; + float dkf_19 = pool_f32[ew_2 + 1536 + d_lane]; + float dgf_19 = pool_f32[ew_2 + 2560 + d_lane]; + float dq_o_19 = dq2_19 + dqf_19; + float dk_o_19 = dk2b_19 + dkf_19 + dkt_3[4]; + float dg_o_19 = __fmaf_rn(q_t_1_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_3[4], k_t_19, cr_18)) + dgf_19; + s_run_2[0] = s_run_2[0] + dg_o_19; +#else + int t_11 = 8 * th + 4; + float r_290[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_11 = r_290[0]; + unsigned int r_291[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_11 = __uint_as_float(r_291[0] << 16); + unsigned int r_292[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_11 = __uint_as_float(r_292[0] << 16); + float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); + float bt_t_11 = btv_0[4]; + float e_off_11 = approx_exp2(g_t_11 - gn_2); + float e_dg_11 = approx_exp2(g_t_11 - gc_3); + float dq2_11 = qdg_0[4] * e_dg_11; + float dk2_11 = kdg_0[4] * e_dg_11; + dbp_2[4] = dk2_11 * k_t_11; + float dk2b_11 = dk2_11 * bt_t_11; + float qe_10 = q_t_1_11 * e_dg_11; + float kp_22 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_11) * kdg_0[4]); + float dqf_11 = pool_f32[ew_2 + 512 + d_lane]; + float dkf_11 = pool_f32[ew_2 + 1536 + d_lane]; + float dgf_11 = pool_f32[ew_2 + 2560 + d_lane]; + float dq_o_11 = dq2_11 + dqf_11; + float dk_o_11 = dk2b_11 + dkf_11 + dkt_3[4]; + float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_3[4], k_t_11, cr_10)) + dgf_11; + s_run_2[0] = s_run_2[0] + dg_o_11; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_27 = t_27 * qrs; - if (t_27 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_19 = t_19 * qrs; + if (t_19 < clen) { +#else + int qoff_11 = t_11 * qrs; + if (t_11 < clen) { +#endif { - auto _gst_493 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_493)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_27; +#if __CUDA_ARCH__ == 1030 + auto _gst_511 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_511)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_511) + (0)) = (__nv_bfloat16)dq_o_19; +#else + auto _gst_497 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_497)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_11; +#endif } { - auto _gst_494 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_494)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_27; +#if __CUDA_ARCH__ == 1030 + auto _gst_512 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_512)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_512) + (0)) = (__nv_bfloat16)dk_o_19; +#else + auto _gst_498 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_498)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_11; +#endif } } - int t_28 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_28 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_28 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_28 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_28 >= clen) ? 0.0f : q_t_28); - float dqf_28 = pool_f32[ew_2 + 384 + d_lane]; - float dkf_28 = pool_f32[ew_2 + 1408 + d_lane]; - float dgf_28 = pool_f32[ew_2 + 2432 + d_lane]; - float bt_t_28 = btv_2[3]; - float e_off_28 = approx_exp2(g_t_28 - gn_2); - float e_dg_28 = approx_exp2(g_t_28 - gc_3); - float dq2_28 = qdg_2[3] * e_dg_28; - float dk2_28 = kdg_2[3] * e_dg_28; - dbp_2[3] = dk2_28 * k_t_28; - float dk2b_28 = dk2_28 * bt_t_28; - float dq_o_28 = dq2_28 + dqf_28; - float dk_o_28 = dk2b_28 + dkf_28 + dkt_3[3]; - float dg_o_28 = __fmaf_rn(q_t_1_28, dq2_28, (dk2b_28 - dkt_3[3]) * k_t_28) + dgf_28; - s_run_2[0] = s_run_2[0] + dg_o_28; +#if __CUDA_ARCH__ == 1030 + int t_20 = 8 * th + 3; + float r_314[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_20 = r_314[0]; + unsigned int r_315[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_20 = __uint_as_float(r_315[0] << 16); + unsigned int r_316[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_20 = __uint_as_float(r_316[0] << 16); + float q_t_1_20 = ((t_20 >= clen) ? 0.0f : q_t_20); + float bt_t_20 = btv_1_1[3]; + float e_off_20 = approx_exp2(g_t_20 - gn_2); + float e_dg_20 = approx_exp2(g_t_20 - gc_3); + float dq2_20 = qdg_1_1[3] * e_dg_20; + float dk2_20 = kdg_1_1[3] * e_dg_20; + dbp_2[3] = dk2_20 * k_t_20; + float dk2b_20 = dk2_20 * bt_t_20; + float qe_19 = q_t_1_20 * e_dg_20; + float kp_31 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1_1[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_20) * kdg_1_1[3]); + float dqf_20 = pool_f32[ew_2 + 384 + d_lane]; + float dkf_20 = pool_f32[ew_2 + 1408 + d_lane]; + float dgf_20 = pool_f32[ew_2 + 2432 + d_lane]; + float dq_o_20 = dq2_20 + dqf_20; + float dk_o_20 = dk2b_20 + dkf_20 + dkt_3[3]; + float dg_o_20 = __fmaf_rn(q_t_1_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_3[3], k_t_20, cr_19)) + dgf_20; + s_run_2[0] = s_run_2[0] + dg_o_20; +#else + int t_12 = 8 * th + 3; + float r_293[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_12 = r_293[0]; + unsigned int r_294[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_12 = __uint_as_float(r_294[0] << 16); + unsigned int r_295[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_12 = __uint_as_float(r_295[0] << 16); + float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); + float bt_t_12 = btv_0[3]; + float e_off_12 = approx_exp2(g_t_12 - gn_2); + float e_dg_12 = approx_exp2(g_t_12 - gc_3); + float dq2_12 = qdg_0[3] * e_dg_12; + float dk2_12 = kdg_0[3] * e_dg_12; + dbp_2[3] = dk2_12 * k_t_12; + float dk2b_12 = dk2_12 * bt_t_12; + float qe_11 = q_t_1_12 * e_dg_12; + float kp_23 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_12) * kdg_0[3]); + float dqf_12 = pool_f32[ew_2 + 384 + d_lane]; + float dkf_12 = pool_f32[ew_2 + 1408 + d_lane]; + float dgf_12 = pool_f32[ew_2 + 2432 + d_lane]; + float dq_o_12 = dq2_12 + dqf_12; + float dk_o_12 = dk2b_12 + dkf_12 + dkt_3[3]; + float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_3[3], k_t_12, cr_11)) + dgf_12; + s_run_2[0] = s_run_2[0] + dg_o_12; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_28 = t_28 * qrs; - if (t_28 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_20 = t_20 * qrs; + if (t_20 < clen) { +#else + int qoff_12 = t_12 * qrs; + if (t_12 < clen) { +#endif { - auto _gst_495 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_495)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_28; +#if __CUDA_ARCH__ == 1030 + auto _gst_513 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_513)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_513) + (0)) = (__nv_bfloat16)dq_o_20; +#else + auto _gst_499 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_499)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_12; +#endif } { - auto _gst_496 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_496)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_28; +#if __CUDA_ARCH__ == 1030 + auto _gst_514 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_514)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_514) + (0)) = (__nv_bfloat16)dk_o_20; +#else + auto _gst_500 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_500)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_12; +#endif } } - int t_29 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_29 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_29 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_29 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_29 >= clen) ? 0.0f : q_t_29); - float dqf_29 = pool_f32[ew_2 + 256 + d_lane]; - float dkf_29 = pool_f32[ew_2 + 1280 + d_lane]; - float dgf_29 = pool_f32[ew_2 + 2304 + d_lane]; - float bt_t_29 = btv_2[2]; - float e_off_29 = approx_exp2(g_t_29 - gn_2); - float e_dg_29 = approx_exp2(g_t_29 - gc_3); - float dq2_29 = qdg_2[2] * e_dg_29; - float dk2_29 = kdg_2[2] * e_dg_29; - dbp_2[2] = dk2_29 * k_t_29; - float dk2b_29 = dk2_29 * bt_t_29; - float dq_o_29 = dq2_29 + dqf_29; - float dk_o_29 = dk2b_29 + dkf_29 + dkt_3[2]; - float dg_o_29 = __fmaf_rn(q_t_1_29, dq2_29, (dk2b_29 - dkt_3[2]) * k_t_29) + dgf_29; - s_run_2[0] = s_run_2[0] + dg_o_29; +#if __CUDA_ARCH__ == 1030 + int t_21 = 8 * th + 2; + float r_317[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_21 = r_317[0]; + unsigned int r_318[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_21 = __uint_as_float(r_318[0] << 16); + unsigned int r_319[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_21 = __uint_as_float(r_319[0] << 16); + float q_t_1_21 = ((t_21 >= clen) ? 0.0f : q_t_21); + float bt_t_21 = btv_1_1[2]; + float e_off_21 = approx_exp2(g_t_21 - gn_2); + float e_dg_21 = approx_exp2(g_t_21 - gc_3); + float dq2_21 = qdg_1_1[2] * e_dg_21; + float dk2_21 = kdg_1_1[2] * e_dg_21; + dbp_2[2] = dk2_21 * k_t_21; + float dk2b_21 = dk2_21 * bt_t_21; + float qe_20 = q_t_1_21 * e_dg_21; + float kp_32 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1_1[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_21) * kdg_1_1[2]); + float dqf_21 = pool_f32[ew_2 + 256 + d_lane]; + float dkf_21 = pool_f32[ew_2 + 1280 + d_lane]; + float dgf_21 = pool_f32[ew_2 + 2304 + d_lane]; + float dq_o_21 = dq2_21 + dqf_21; + float dk_o_21 = dk2b_21 + dkf_21 + dkt_3[2]; + float dg_o_21 = __fmaf_rn(q_t_1_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_3[2], k_t_21, cr_20)) + dgf_21; + s_run_2[0] = s_run_2[0] + dg_o_21; +#else + int t_13 = 8 * th + 2; + float r_296[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_13 = r_296[0]; + unsigned int r_297[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_13 = __uint_as_float(r_297[0] << 16); + unsigned int r_298[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_13 = __uint_as_float(r_298[0] << 16); + float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); + float bt_t_13 = btv_0[2]; + float e_off_13 = approx_exp2(g_t_13 - gn_2); + float e_dg_13 = approx_exp2(g_t_13 - gc_3); + float dq2_13 = qdg_0[2] * e_dg_13; + float dk2_13 = kdg_0[2] * e_dg_13; + dbp_2[2] = dk2_13 * k_t_13; + float dk2b_13 = dk2_13 * bt_t_13; + float qe_12 = q_t_1_13 * e_dg_13; + float kp_24 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_13) * kdg_0[2]); + float dqf_13 = pool_f32[ew_2 + 256 + d_lane]; + float dkf_13 = pool_f32[ew_2 + 1280 + d_lane]; + float dgf_13 = pool_f32[ew_2 + 2304 + d_lane]; + float dq_o_13 = dq2_13 + dqf_13; + float dk_o_13 = dk2b_13 + dkf_13 + dkt_3[2]; + float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_3[2], k_t_13, cr_12)) + dgf_13; + s_run_2[0] = s_run_2[0] + dg_o_13; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_29 = t_29 * qrs; - if (t_29 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_21 = t_21 * qrs; + if (t_21 < clen) { +#else + int qoff_13 = t_13 * qrs; + if (t_13 < clen) { +#endif { - auto _gst_497 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_497)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_29; +#if __CUDA_ARCH__ == 1030 + auto _gst_515 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_515)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_515) + (0)) = (__nv_bfloat16)dq_o_21; +#else + auto _gst_501 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_501)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_13; +#endif } { - auto _gst_498 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_498)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_29; +#if __CUDA_ARCH__ == 1030 + auto _gst_516 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_516)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_516) + (0)) = (__nv_bfloat16)dk_o_21; +#else + auto _gst_502 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_502)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_13; +#endif } } - int t_30 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_30 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_30 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_30 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_30 >= clen) ? 0.0f : q_t_30); - float dqf_30 = pool_f32[ew_2 + 128 + d_lane]; - float dkf_30 = pool_f32[ew_2 + 1152 + d_lane]; - float dgf_30 = pool_f32[ew_2 + 2176 + d_lane]; - float bt_t_30 = btv_2[1]; - float e_off_30 = approx_exp2(g_t_30 - gn_2); - float e_dg_30 = approx_exp2(g_t_30 - gc_3); - float dq2_30 = qdg_2[1] * e_dg_30; - float dk2_30 = kdg_2[1] * e_dg_30; - dbp_2[1] = dk2_30 * k_t_30; - float dk2b_30 = dk2_30 * bt_t_30; - float dq_o_30 = dq2_30 + dqf_30; - float dk_o_30 = dk2b_30 + dkf_30 + dkt_3[1]; - float dg_o_30 = __fmaf_rn(q_t_1_30, dq2_30, (dk2b_30 - dkt_3[1]) * k_t_30) + dgf_30; - s_run_2[0] = s_run_2[0] + dg_o_30; +#if __CUDA_ARCH__ == 1030 + int t_22 = 8 * th + 1; + float r_320[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_22 = r_320[0]; + unsigned int r_321[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_22 = __uint_as_float(r_321[0] << 16); + unsigned int r_322[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_22 = __uint_as_float(r_322[0] << 16); + float q_t_1_22 = ((t_22 >= clen) ? 0.0f : q_t_22); + float bt_t_22 = btv_1_1[1]; + float e_off_22 = approx_exp2(g_t_22 - gn_2); + float e_dg_22 = approx_exp2(g_t_22 - gc_3); + float dq2_22 = qdg_1_1[1] * e_dg_22; + float dk2_22 = kdg_1_1[1] * e_dg_22; + dbp_2[1] = dk2_22 * k_t_22; + float dk2b_22 = dk2_22 * bt_t_22; + float qe_21 = q_t_1_22 * e_dg_22; + float kp_33 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1_1[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_22) * kdg_1_1[1]); + float dqf_22 = pool_f32[ew_2 + 128 + d_lane]; + float dkf_22 = pool_f32[ew_2 + 1152 + d_lane]; + float dgf_22 = pool_f32[ew_2 + 2176 + d_lane]; + float dq_o_22 = dq2_22 + dqf_22; + float dk_o_22 = dk2b_22 + dkf_22 + dkt_3[1]; + float dg_o_22 = __fmaf_rn(q_t_1_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_3[1], k_t_22, cr_21)) + dgf_22; + s_run_2[0] = s_run_2[0] + dg_o_22; +#else + int t_14 = 8 * th + 1; + float r_299[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_14 = r_299[0]; + unsigned int r_300[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_14 = __uint_as_float(r_300[0] << 16); + unsigned int r_301[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_14 = __uint_as_float(r_301[0] << 16); + float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); + float bt_t_14 = btv_0[1]; + float e_off_14 = approx_exp2(g_t_14 - gn_2); + float e_dg_14 = approx_exp2(g_t_14 - gc_3); + float dq2_14 = qdg_0[1] * e_dg_14; + float dk2_14 = kdg_0[1] * e_dg_14; + dbp_2[1] = dk2_14 * k_t_14; + float dk2b_14 = dk2_14 * bt_t_14; + float qe_13 = q_t_1_14 * e_dg_14; + float kp_25 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_14) * kdg_0[1]); + float dqf_14 = pool_f32[ew_2 + 128 + d_lane]; + float dkf_14 = pool_f32[ew_2 + 1152 + d_lane]; + float dgf_14 = pool_f32[ew_2 + 2176 + d_lane]; + float dq_o_14 = dq2_14 + dqf_14; + float dk_o_14 = dk2b_14 + dkf_14 + dkt_3[1]; + float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_3[1], k_t_14, cr_13)) + dgf_14; + s_run_2[0] = s_run_2[0] + dg_o_14; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_30 = t_30 * qrs; - if (t_30 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_22 = t_22 * qrs; + if (t_22 < clen) { +#else + int qoff_14 = t_14 * qrs; + if (t_14 < clen) { +#endif { - auto _gst_499 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_499)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_30; +#if __CUDA_ARCH__ == 1030 + auto _gst_517 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_517)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_517) + (0)) = (__nv_bfloat16)dq_o_22; +#else + auto _gst_503 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_503)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_503) + (0)) = (__nv_bfloat16)dq_o_14; +#endif } { - auto _gst_500 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_500)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_30; +#if __CUDA_ARCH__ == 1030 + auto _gst_518 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_518)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_518) + (0)) = (__nv_bfloat16)dk_o_22; +#else + auto _gst_504 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_504)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_504) + (0)) = (__nv_bfloat16)dk_o_14; +#endif } } - int t_31 = 0 + 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_31 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_31 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_31 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_31 >= clen) ? 0.0f : q_t_31); - float dqf_31 = pool_f32[ew_2 + d_lane]; - float dkf_31 = pool_f32[ew_2 + 1024 + d_lane]; - float dgf_31 = pool_f32[ew_2 + 2048 + d_lane]; - float bt_t_31 = btv_2[0]; - float e_off_31 = approx_exp2(g_t_31 - gn_2); - float e_dg_31 = approx_exp2(g_t_31 - gc_3); - float dq2_31 = qdg_2[0] * e_dg_31; - float dk2_31 = kdg_2[0] * e_dg_31; - dbp_2[0] = dk2_31 * k_t_31; - float dk2b_31 = dk2_31 * bt_t_31; - float dq_o_31 = dq2_31 + dqf_31; - float dk_o_31 = dk2b_31 + dkf_31 + dkt_3[0]; - float dg_o_31 = __fmaf_rn(q_t_1_31, dq2_31, (dk2b_31 - dkt_3[0]) * k_t_31) + dgf_31; - s_run_2[0] = s_run_2[0] + dg_o_31; +#if __CUDA_ARCH__ == 1030 + int t_23 = 0 + 8 * th; + float r_323[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_23 = r_323[0]; + unsigned int r_324[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_23 = __uint_as_float(r_324[0] << 16); + unsigned int r_325[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_23 = __uint_as_float(r_325[0] << 16); + float q_t_1_23 = ((t_23 >= clen) ? 0.0f : q_t_23); + float bt_t_23 = btv_1_1[0]; + float e_off_23 = approx_exp2(g_t_23 - gn_2); + float e_dg_23 = approx_exp2(g_t_23 - gc_3); + float dq2_23 = qdg_1_1[0] * e_dg_23; + float dk2_23 = kdg_1_1[0] * e_dg_23; + dbp_2[0] = dk2_23 * k_t_23; + float dk2b_23 = dk2_23 * bt_t_23; + float qe_22 = q_t_1_23 * e_dg_23; + float kp_34 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1_1[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_23) * kdg_1_1[0]); + float dqf_23 = pool_f32[ew_2 + d_lane]; + float dkf_23 = pool_f32[ew_2 + 1024 + d_lane]; + float dgf_23 = pool_f32[ew_2 + 2048 + d_lane]; + float dq_o_23 = dq2_23 + dqf_23; + float dk_o_23 = dk2b_23 + dkf_23 + dkt_3[0]; + float dg_o_23 = __fmaf_rn(q_t_1_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_3[0], k_t_23, cr_22)) + dgf_23; + s_run_2[0] = s_run_2[0] + dg_o_23; +#else + int t_15 = 0 + 8 * th; + float r_302[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_15 = r_302[0]; + unsigned int r_303[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_15 = __uint_as_float(r_303[0] << 16); + unsigned int r_304[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_15 = __uint_as_float(r_304[0] << 16); + float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); + float bt_t_15 = btv_0[0]; + float e_off_15 = approx_exp2(g_t_15 - gn_2); + float e_dg_15 = approx_exp2(g_t_15 - gc_3); + float dq2_15 = qdg_0[0] * e_dg_15; + float dk2_15 = kdg_0[0] * e_dg_15; + dbp_2[0] = dk2_15 * k_t_15; + float dk2b_15 = dk2_15 * bt_t_15; + float qe_14 = q_t_1_15 * e_dg_15; + float kp_26 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_15) * kdg_0[0]); + float dqf_15 = pool_f32[ew_2 + d_lane]; + float dkf_15 = pool_f32[ew_2 + 1024 + d_lane]; + float dgf_15 = pool_f32[ew_2 + 2048 + d_lane]; + float dq_o_15 = dq2_15 + dqf_15; + float dk_o_15 = dk2b_15 + dkf_15 + dkt_3[0]; + float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_3[0], k_t_15, cr_14)) + dgf_15; + s_run_2[0] = s_run_2[0] + dg_o_15; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_31 = t_31 * qrs; - if (t_31 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_23 = t_23 * qrs; + if (t_23 < clen) { +#else + int qoff_15 = t_15 * qrs; + if (t_15 < clen) { +#endif { - auto _gst_501 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_501)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_31; +#if __CUDA_ARCH__ == 1030 + auto _gst_519 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_519)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_519) + (0)) = (__nv_bfloat16)dq_o_23; +#else + auto _gst_505 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_505)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_505) + (0)) = (__nv_bfloat16)dq_o_15; +#endif } { - auto _gst_502 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_502)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_31; +#if __CUDA_ARCH__ == 1030 + auto _gst_520 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_520)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_520) + (0)) = (__nv_bfloat16)dk_o_23; +#else + auto _gst_506 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_506)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_506) + (0)) = (__nv_bfloat16)dk_o_15; +#endif } } float tot_2 = s_run_2[0]; @@ -5626,37 +7387,81 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_503 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_503; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_521 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_521; +#else + float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_507; +#endif float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_504; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_522 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_522; +#else + float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_508; +#endif float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_505 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_505; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_523 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_523; +#else + float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_509; +#endif float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_506; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_524 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_524; +#else + float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_510; +#endif float v2_18[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_18[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_507; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_525 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_18[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_525; +#else + float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_18[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_511; +#endif float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_18[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_508; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_526 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_18[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_526; +#else + float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_18[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_512; +#endif float send_26 = ((b2_2 == 0) ? v2_18[1] : v2_18[0]); - float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_18[0] : v2_18[1]) + _shfl_xor_sync_509; - float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_510; - float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_511; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_527 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_18[0] : v2_18[1]) + _shfl_xor_sync_527; + float _shfl_xor_sync_528 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_528; + float _shfl_xor_sync_529 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_529; +#else + float _shfl_xor_sync_513 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_18[0] : v2_18[1]) + _shfl_xor_sync_513; + float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_514; + float _shfl_xor_sync_515 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_515; +#endif tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_512 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_512), "f"(tot_2) : "memory"); +#if __CUDA_ARCH__ == 1030 + uint32_t _addr_530 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_530), "f"(tot_2) : "memory"); +#else + uint32_t _addr_516 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_516), "f"(tot_2) : "memory"); +#endif } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5688,132 +7493,280 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_513 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_531 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_517 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_513), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_516; - asm("abs.f32 %0, %1;" : "=f"(_abs_516) : "f"(rp_z_11.x)); - float _neg_515; - asm("neg.f32 %0, %1;" : "=f"(_neg_515) : "f"(_abs_516)); - float _exp_514 = __nv_expf(_neg_515); - float _abs_519; - asm("abs.f32 %0, %1;" : "=f"(_abs_519) : "f"(rp_z_11.y)); - float _neg_518; - asm("neg.f32 %0, %1;" : "=f"(_neg_518) : "f"(_abs_519)); - float _exp_517 = __nv_expf(_neg_518); - float2 rp_e_11 = make_float2(_exp_514, _exp_517); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_531), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_534; + asm("abs.f32 %0, %1;" : "=f"(_abs_534) : "f"(rp_z_11.x)); + float _neg_533; + asm("neg.f32 %0, %1;" : "=f"(_neg_533) : "f"(_abs_534)); + float _exp_532 = __nv_expf(_neg_533); + float _abs_537; + asm("abs.f32 %0, %1;" : "=f"(_abs_537) : "f"(rp_z_11.y)); + float _neg_536; + asm("neg.f32 %0, %1;" : "=f"(_neg_536) : "f"(_abs_537)); + float _exp_535 = __nv_expf(_neg_536); + float2 rp_e_11 = make_float2(_exp_532, _exp_535); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_517), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_520; + asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_11.x)); + float _neg_519; + asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); + float _exp_518 = __nv_expf(_neg_519); + float _abs_523; + asm("abs.f32 %0, %1;" : "=f"(_abs_523) : "f"(rp_z_11.y)); + float _neg_522; + asm("neg.f32 %0, %1;" : "=f"(_neg_522) : "f"(_abs_523)); + float _exp_521 = __nv_expf(_neg_522); + float2 rp_e_11 = make_float2(_exp_518, _exp_521); +#endif float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_520 = approx_rcp(rp_den_11.x); - float _rcp_521 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_522 = make_float2(_rcp_520, _rcp_521); +#if __CUDA_ARCH__ == 1030 + float _rcp_538 = approx_rcp(rp_den_11.x); + float _rcp_539 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_540 = make_float2(_rcp_538, _rcp_539); +#else + float _rcp_524 = approx_rcp(rp_den_11.x); + float _rcp_525 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_526 = make_float2(_rcp_524, _rcp_525); +#endif float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_522)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_540)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_526)); +#endif float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_523 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_541 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_527 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_541)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_527)); +#endif u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_524 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_542 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_528 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_524), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_527; - asm("abs.f32 %0, %1;" : "=f"(_abs_527) : "f"(rp_z_12.x)); - float _neg_526; - asm("neg.f32 %0, %1;" : "=f"(_neg_526) : "f"(_abs_527)); - float _exp_525 = __nv_expf(_neg_526); - float _abs_530; - asm("abs.f32 %0, %1;" : "=f"(_abs_530) : "f"(rp_z_12.y)); - float _neg_529; - asm("neg.f32 %0, %1;" : "=f"(_neg_529) : "f"(_abs_530)); - float _exp_528 = __nv_expf(_neg_529); - float2 rp_e_12 = make_float2(_exp_525, _exp_528); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_542), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_545; + asm("abs.f32 %0, %1;" : "=f"(_abs_545) : "f"(rp_z_12.x)); + float _neg_544; + asm("neg.f32 %0, %1;" : "=f"(_neg_544) : "f"(_abs_545)); + float _exp_543 = __nv_expf(_neg_544); + float _abs_548; + asm("abs.f32 %0, %1;" : "=f"(_abs_548) : "f"(rp_z_12.y)); + float _neg_547; + asm("neg.f32 %0, %1;" : "=f"(_neg_547) : "f"(_abs_548)); + float _exp_546 = __nv_expf(_neg_547); + float2 rp_e_12 = make_float2(_exp_543, _exp_546); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_528), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_531; + asm("abs.f32 %0, %1;" : "=f"(_abs_531) : "f"(rp_z_12.x)); + float _neg_530; + asm("neg.f32 %0, %1;" : "=f"(_neg_530) : "f"(_abs_531)); + float _exp_529 = __nv_expf(_neg_530); + float _abs_534; + asm("abs.f32 %0, %1;" : "=f"(_abs_534) : "f"(rp_z_12.y)); + float _neg_533; + asm("neg.f32 %0, %1;" : "=f"(_neg_533) : "f"(_abs_534)); + float _exp_532 = __nv_expf(_neg_533); + float2 rp_e_12 = make_float2(_exp_529, _exp_532); +#endif float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_531 = approx_rcp(rp_den_12.x); - float _rcp_532 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_533 = make_float2(_rcp_531, _rcp_532); +#if __CUDA_ARCH__ == 1030 + float _rcp_549 = approx_rcp(rp_den_12.x); + float _rcp_550 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_551 = make_float2(_rcp_549, _rcp_550); +#else + float _rcp_535 = approx_rcp(rp_den_12.x); + float _rcp_536 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_537 = make_float2(_rcp_535, _rcp_536); +#endif float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_533)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_551)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_537)); +#endif float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_534 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_552 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_538 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_534)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_552)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_538)); +#endif u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_535 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_553 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_539 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_535), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_538; - asm("abs.f32 %0, %1;" : "=f"(_abs_538) : "f"(rp_z_13.x)); - float _neg_537; - asm("neg.f32 %0, %1;" : "=f"(_neg_537) : "f"(_abs_538)); - float _exp_536 = __nv_expf(_neg_537); - float _abs_541; - asm("abs.f32 %0, %1;" : "=f"(_abs_541) : "f"(rp_z_13.y)); - float _neg_540; - asm("neg.f32 %0, %1;" : "=f"(_neg_540) : "f"(_abs_541)); - float _exp_539 = __nv_expf(_neg_540); - float2 rp_e_13 = make_float2(_exp_536, _exp_539); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_553), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_556; + asm("abs.f32 %0, %1;" : "=f"(_abs_556) : "f"(rp_z_13.x)); + float _neg_555; + asm("neg.f32 %0, %1;" : "=f"(_neg_555) : "f"(_abs_556)); + float _exp_554 = __nv_expf(_neg_555); + float _abs_559; + asm("abs.f32 %0, %1;" : "=f"(_abs_559) : "f"(rp_z_13.y)); + float _neg_558; + asm("neg.f32 %0, %1;" : "=f"(_neg_558) : "f"(_abs_559)); + float _exp_557 = __nv_expf(_neg_558); + float2 rp_e_13 = make_float2(_exp_554, _exp_557); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_539), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_542; + asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(rp_z_13.x)); + float _neg_541; + asm("neg.f32 %0, %1;" : "=f"(_neg_541) : "f"(_abs_542)); + float _exp_540 = __nv_expf(_neg_541); + float _abs_545; + asm("abs.f32 %0, %1;" : "=f"(_abs_545) : "f"(rp_z_13.y)); + float _neg_544; + asm("neg.f32 %0, %1;" : "=f"(_neg_544) : "f"(_abs_545)); + float _exp_543 = __nv_expf(_neg_544); + float2 rp_e_13 = make_float2(_exp_540, _exp_543); +#endif float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_542 = approx_rcp(rp_den_13.x); - float _rcp_543 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_544 = make_float2(_rcp_542, _rcp_543); +#if __CUDA_ARCH__ == 1030 + float _rcp_560 = approx_rcp(rp_den_13.x); + float _rcp_561 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_562 = make_float2(_rcp_560, _rcp_561); +#else + float _rcp_546 = approx_rcp(rp_den_13.x); + float _rcp_547 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_548 = make_float2(_rcp_546, _rcp_547); +#endif float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_544)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_562)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_548)); +#endif float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_545 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_563 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_549 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_545)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_563)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_549)); +#endif u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_546 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_564 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_550 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_546), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_549; - asm("abs.f32 %0, %1;" : "=f"(_abs_549) : "f"(rp_z_14.x)); - float _neg_548; - asm("neg.f32 %0, %1;" : "=f"(_neg_548) : "f"(_abs_549)); - float _exp_547 = __nv_expf(_neg_548); - float _abs_552; - asm("abs.f32 %0, %1;" : "=f"(_abs_552) : "f"(rp_z_14.y)); - float _neg_551; - asm("neg.f32 %0, %1;" : "=f"(_neg_551) : "f"(_abs_552)); - float _exp_550 = __nv_expf(_neg_551); - float2 rp_e_14 = make_float2(_exp_547, _exp_550); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_564), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_567; + asm("abs.f32 %0, %1;" : "=f"(_abs_567) : "f"(rp_z_14.x)); + float _neg_566; + asm("neg.f32 %0, %1;" : "=f"(_neg_566) : "f"(_abs_567)); + float _exp_565 = __nv_expf(_neg_566); + float _abs_570; + asm("abs.f32 %0, %1;" : "=f"(_abs_570) : "f"(rp_z_14.y)); + float _neg_569; + asm("neg.f32 %0, %1;" : "=f"(_neg_569) : "f"(_abs_570)); + float _exp_568 = __nv_expf(_neg_569); + float2 rp_e_14 = make_float2(_exp_565, _exp_568); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_550), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_553; + asm("abs.f32 %0, %1;" : "=f"(_abs_553) : "f"(rp_z_14.x)); + float _neg_552; + asm("neg.f32 %0, %1;" : "=f"(_neg_552) : "f"(_abs_553)); + float _exp_551 = __nv_expf(_neg_552); + float _abs_556; + asm("abs.f32 %0, %1;" : "=f"(_abs_556) : "f"(rp_z_14.y)); + float _neg_555; + asm("neg.f32 %0, %1;" : "=f"(_neg_555) : "f"(_abs_556)); + float _exp_554 = __nv_expf(_neg_555); + float2 rp_e_14 = make_float2(_exp_551, _exp_554); +#endif float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_553 = approx_rcp(rp_den_14.x); - float _rcp_554 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_555 = make_float2(_rcp_553, _rcp_554); +#if __CUDA_ARCH__ == 1030 + float _rcp_571 = approx_rcp(rp_den_14.x); + float _rcp_572 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_573 = make_float2(_rcp_571, _rcp_572); +#else + float _rcp_557 = approx_rcp(rp_den_14.x); + float _rcp_558 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_559 = make_float2(_rcp_557, _rcp_558); +#endif float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_555)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_573)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_559)); +#endif float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_556 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_574 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_560 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_556)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_574)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_560)); +#endif u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -5828,54 +7781,102 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_557; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_557) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_557; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_575; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_575) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_575; +#else + uint32_t _prmt_561; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_561; +#endif int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_558 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_558)); - *(reinterpret_cast(_gst_558) + (0)) = pq_12[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_576 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_576)); + *(reinterpret_cast(_gst_576) + (0)) = pq_12[0]; +#else + auto _gst_562 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_562)); + *(reinterpret_cast(_gst_562) + (0)) = pq_12[0]; +#endif } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_559; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_559) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_559; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_577; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_577) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_577; +#else + uint32_t _prmt_563; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_563; +#endif int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_560 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_560)); - *(reinterpret_cast(_gst_560) + (0)) = pq_13[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_578 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_578)); + *(reinterpret_cast(_gst_578) + (0)) = pq_13[0]; +#else + auto _gst_564 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_564)); + *(reinterpret_cast(_gst_564) + (0)) = pq_13[0]; +#endif } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_561; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_561; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_579; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_579) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_579; +#else + uint32_t _prmt_565; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_565) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_565; +#endif int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_562 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_562)); - *(reinterpret_cast(_gst_562) + (0)) = pq_14[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_580 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_580)); + *(reinterpret_cast(_gst_580) + (0)) = pq_14[0]; +#else + auto _gst_566 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_566)); + *(reinterpret_cast(_gst_566) + (0)) = pq_14[0]; +#endif } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_563; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_563; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_581; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_581) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_581; +#else + uint32_t _prmt_567; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_567) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_567; +#endif int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_564 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_564)); - *(reinterpret_cast(_gst_564) + (0)) = pq_15[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_582 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_582)); + *(reinterpret_cast(_gst_582) + (0)) = pq_15[0]; +#else + auto _gst_568 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_568)); + *(reinterpret_cast(_gst_568) + (0)) = pq_15[0]; +#endif } } int t_a_24 = 8 * th; @@ -5940,47 +7941,99 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_565; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_565) : "f"(s_a_2), "f"(nm_t_5)); +#if __CUDA_ARCH__ == 1030 + float _sub_583; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(s_a_2), "f"(nm_t_5)); +#else + float _sub_569; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_569) : "f"(s_a_2), "f"(nm_t_5)); +#endif float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_565), "f"(pa_2)); - float _sub_566; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_566) : "f"(pa_2), "f"(nm_t_5)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_583), "f"(pa_2)); + float _sub_584; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_584) : "f"(pa_2), "f"(nm_t_5)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_569), "f"(pa_2)); + float _sub_570; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_570) : "f"(pa_2), "f"(nm_t_5)); +#endif float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_566), "f"(s_a_2)); - float _abs_568; - asm("abs.f32 %0, %1;" : "=f"(_abs_568) : "f"(s_a_2)); - float _abs_569; - asm("abs.f32 %0, %1;" : "=f"(_abs_569) : "f"(pa_2)); - uint32_t _setp_567_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_567_bits) : "f"(_abs_568), "f"(_abs_569)); - bool _setp_567 = _setp_567_bits != 0; - float nm_err_5 = ((_setp_567) ? nm_es_5 : nm_ex_5); - float _add_570; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_570; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_584), "f"(s_a_2)); + float _abs_586; + asm("abs.f32 %0, %1;" : "=f"(_abs_586) : "f"(s_a_2)); + float _abs_587; + asm("abs.f32 %0, %1;" : "=f"(_abs_587) : "f"(pa_2)); + uint32_t _setp_585_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_585_bits) : "f"(_abs_586), "f"(_abs_587)); + bool _setp_585 = _setp_585_bits != 0; + float nm_err_5 = ((_setp_585) ? nm_es_5 : nm_ex_5); + float _add_588; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_588) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_588; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_570), "f"(s_a_2)); + float _abs_572; + asm("abs.f32 %0, %1;" : "=f"(_abs_572) : "f"(s_a_2)); + float _abs_573; + asm("abs.f32 %0, %1;" : "=f"(_abs_573) : "f"(pa_2)); + uint32_t _setp_571_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_571_bits) : "f"(_abs_572), "f"(_abs_573)); + bool _setp_571 = _setp_571_bits != 0; + float nm_err_5 = ((_setp_571) ? nm_es_5 : nm_ex_5); + float _add_574; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_574; +#endif s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_571; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(s_b_2), "f"(nm_t_6)); +#if __CUDA_ARCH__ == 1030 + float _sub_589; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(s_b_2), "f"(nm_t_6)); +#else + float _sub_575; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_575) : "f"(s_b_2), "f"(nm_t_6)); +#endif float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_571), "f"(pb_2)); - float _sub_572; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_572) : "f"(pb_2), "f"(nm_t_6)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_589), "f"(pb_2)); + float _sub_590; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(pb_2), "f"(nm_t_6)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_575), "f"(pb_2)); + float _sub_576; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_576) : "f"(pb_2), "f"(nm_t_6)); +#endif float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_572), "f"(s_b_2)); - float _abs_574; - asm("abs.f32 %0, %1;" : "=f"(_abs_574) : "f"(s_b_2)); - float _abs_575; - asm("abs.f32 %0, %1;" : "=f"(_abs_575) : "f"(pb_2)); - uint32_t _setp_573_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_573_bits) : "f"(_abs_574), "f"(_abs_575)); - bool _setp_573 = _setp_573_bits != 0; - float nm_err_6 = ((_setp_573) ? nm_es_6 : nm_ex_6); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_576; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_590), "f"(s_b_2)); + float _abs_592; + asm("abs.f32 %0, %1;" : "=f"(_abs_592) : "f"(s_b_2)); + float _abs_593; + asm("abs.f32 %0, %1;" : "=f"(_abs_593) : "f"(pb_2)); + uint32_t _setp_591_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_591_bits) : "f"(_abs_592), "f"(_abs_593)); + bool _setp_591 = _setp_591_bits != 0; + float nm_err_6 = ((_setp_591) ? nm_es_6 : nm_ex_6); + float _add_594; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_594) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_594; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_576), "f"(s_b_2)); + float _abs_578; + asm("abs.f32 %0, %1;" : "=f"(_abs_578) : "f"(s_b_2)); + float _abs_579; + asm("abs.f32 %0, %1;" : "=f"(_abs_579) : "f"(pb_2)); + uint32_t _setp_577_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_577_bits) : "f"(_abs_578), "f"(_abs_579)); + bool _setp_577 = _setp_577_bits != 0; + float nm_err_6 = ((_setp_577) ? nm_es_6 : nm_ex_6); + float _add_580; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_580; +#endif s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -6019,60 +8072,124 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_578; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_578) : "f"(ts_s), "f"(ts_bb)); - float _sub_577; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_577) : "f"(s_a_3), "f"(_sub_578)); - float _sub_579; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_579) : "f"(other[0]), "f"(ts_bb)); +#if __CUDA_ARCH__ == 1030 + float _sub_596; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_596) : "f"(ts_s), "f"(ts_bb)); + float _sub_595; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_595) : "f"(s_a_3), "f"(_sub_596)); + float _sub_597; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_597) : "f"(other[0]), "f"(ts_bb)); +#else + float _sub_582; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s), "f"(ts_bb)); + float _sub_581; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(s_a_3), "f"(_sub_582)); + float _sub_583; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(other[0]), "f"(ts_bb)); +#endif float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_577), "f"(_sub_579)); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_a_3), "f"(other[1])); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_595), "f"(_sub_597)); + float _add_598; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_598) : "f"(c_a_3), "f"(other[1])); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_581), "f"(_sub_583)); + float _add_584; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(c_a_3), "f"(other[1])); +#endif float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_580), "f"(ts_e)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_598), "f"(ts_e)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_584), "f"(ts_e)); +#endif float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_582; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_581; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(ts_s), "f"(_sub_582)); - float _sub_583; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(pm_c), "f"(ts_bb_0)); +#if __CUDA_ARCH__ == 1030 + float _sub_600; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_600) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_599; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_599) : "f"(ts_s), "f"(_sub_600)); + float _sub_601; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_601) : "f"(pm_c), "f"(ts_bb_0)); +#else + float _sub_586; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_585; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s), "f"(_sub_586)); + float _sub_587; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_587) : "f"(pm_c), "f"(ts_bb_0)); +#endif float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_581), "f"(_sub_583)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_599), "f"(_sub_601)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_585), "f"(_sub_587)); +#endif s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_585; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_584; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_584) : "f"(s_b_3), "f"(_sub_585)); - float _sub_586; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(other[2]), "f"(ts_bb_1)); +#if __CUDA_ARCH__ == 1030 + float _sub_603; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_603) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_602; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_602) : "f"(s_b_3), "f"(_sub_603)); + float _sub_604; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_604) : "f"(other[2]), "f"(ts_bb_1)); +#else + float _sub_589; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_588; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(s_b_3), "f"(_sub_589)); + float _sub_590; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(other[2]), "f"(ts_bb_1)); +#endif float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_584), "f"(_sub_586)); - float _add_587; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_587) : "f"(c_b_3), "f"(other[3])); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_602), "f"(_sub_604)); + float _add_605; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_605) : "f"(c_b_3), "f"(other[3])); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_588), "f"(_sub_590)); + float _add_591; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_591) : "f"(c_b_3), "f"(other[3])); +#endif float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_587), "f"(ts_e_1)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_605), "f"(ts_e_1)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_591), "f"(ts_e_1)); +#endif float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_589; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_588; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(ts_s_1), "f"(_sub_589)); - float _sub_590; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(pm_c_0), "f"(ts_bb_2)); +#if __CUDA_ARCH__ == 1030 + float _sub_607; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_607) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_606; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_606) : "f"(ts_s_1), "f"(_sub_607)); + float _sub_608; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_608) : "f"(pm_c_0), "f"(ts_bb_2)); +#else + float _sub_593; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_593) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_592; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_592) : "f"(ts_s_1), "f"(_sub_593)); + float _sub_594; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_594) : "f"(pm_c_0), "f"(ts_bb_2)); +#endif float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_588), "f"(_sub_590)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_606), "f"(_sub_608)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_592), "f"(_sub_594)); +#endif s_b_3 = ts_s_2; c_b_3 = ts_e_2; double v_a = (double)s_a_3 + (double)c_a_3; @@ -6175,41 +8292,79 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); +#if __CUDA_ARCH__ == 1030 + for (int r_326 = 0; r_326 < 16; r_326++) { + int t_24 = r_326; + if (t_24 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); +#else + for (int r_305 = 0; r_305 < 16; r_305++) { + int t_16 = r_305; + if (t_16 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); +#endif } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6276,41 +8431,79 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_32 = 16 + r_348; - if (t_32 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); +#if __CUDA_ARCH__ == 1030 + for (int r_327 = 0; r_327 < 16; r_327++) { + int t_25 = 16 + r_327; + if (t_25 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); +#else + for (int r_306 = 0; r_306 < 16; r_306++) { + int t_17 = 16 + r_306; + if (t_17 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); +#endif } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6416,41 +8609,79 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_33 = 32 + r_349; - if (t_33 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); +#if __CUDA_ARCH__ == 1030 + for (int r_328 = 0; r_328 < 16; r_328++) { + int t_26 = 32 + r_328; + if (t_26 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); +#else + for (int r_307 = 0; r_307 < 16; r_307++) { + int t_18 = 32 + r_307; + if (t_18 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); +#endif } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6583,41 +8814,79 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_34 = 48 + r_350; - if (t_34 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); +#if __CUDA_ARCH__ == 1030 + for (int r_329 = 0; r_329 < 16; r_329++) { + int t_27 = 48 + r_329; + if (t_27 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); +#else + for (int r_308 = 0; r_308 < 16; r_308++) { + int t_19 = 48 + r_308; + if (t_19 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); +#endif } } asm volatile("tcgen05.fence::before_thread_sync;"); diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu index af6add58ac8..59b5fcea001 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2246,15 +2269,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2275,15 +2304,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2304,15 +2339,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2333,15 +2374,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2502,15 +2573,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2531,15 +2608,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2560,15 +2643,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2589,15 +2678,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2636,15 +2731,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2665,15 +2766,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2694,15 +2801,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2723,15 +2836,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2802,37 +2921,37 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2967,7 +3086,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[7]; float q_r_0 = q_t0_0; - float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, (dk2b_0 - dkt_3[7]) * k_t_0)) + dgf_0; + float qe = q_r_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[7]); + float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[7], k_t_0, cr))) + dgf_0; s_run[0] = s_run[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -2977,14 +3101,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_248 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_249 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 6; @@ -3011,7 +3135,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[6]; float q_r_1 = q_t0_1; - float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, (dk2b_1 - dkt_3[6]) * k_t_1)) + dgf_1; + float qe_0 = q_r_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[6]); + float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[6], k_t_1, cr_0))) + dgf_1; s_run[0] = s_run[0] + dg_o_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3021,14 +3150,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_1 = t_1 * qrs; if (t_1 < clen) { { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_250 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_251 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 5; @@ -3055,7 +3184,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[5]; float q_r_2 = q_t0_2; - float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, (dk2b_2 - dkt_3[5]) * k_t_2)) + dgf_2; + float qe_1 = q_r_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[5]); + float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[5], k_t_2, cr_1))) + dgf_2; s_run[0] = s_run[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3065,14 +3199,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_252 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_253 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 4; @@ -3099,7 +3233,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[4]; float q_r_3 = q_t0_3; - float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, (dk2b_3 - dkt_3[4]) * k_t_3)) + dgf_3; + float qe_2 = q_r_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[4]); + float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[4], k_t_3, cr_2))) + dgf_3; s_run[0] = s_run[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3109,14 +3248,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_254 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_255 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 3; @@ -3143,7 +3282,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[3]; float q_r_4 = q_t0_4; - float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, (dk2b_4 - dkt_3[3]) * k_t_4)) + dgf_4; + float qe_3 = q_r_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[3]); + float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[3], k_t_4, cr_3))) + dgf_4; s_run[0] = s_run[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3153,14 +3297,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_256 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_257 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 2; @@ -3187,7 +3331,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[2]; float q_r_5 = q_t0_5; - float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, (dk2b_5 - dkt_3[2]) * k_t_5)) + dgf_5; + float qe_4 = q_r_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[2]); + float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[2], k_t_5, cr_4))) + dgf_5; s_run[0] = s_run[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3197,14 +3346,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_258 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_259 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 1; @@ -3231,7 +3380,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[1]; float q_r_6 = q_t0_6; - float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, (dk2b_6 - dkt_3[1]) * k_t_6)) + dgf_6; + float qe_5 = q_r_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[1]); + float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[1], k_t_6, cr_5))) + dgf_6; s_run[0] = s_run[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3241,14 +3395,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_260 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_261 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th; @@ -3275,7 +3429,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[0]; float q_r_7 = q_t0_7; - float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, (dk2b_7 - dkt_3[0]) * k_t_7)) + dgf_7; + float qe_6 = q_r_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[0]); + float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[0], k_t_7, cr_6))) + dgf_7; s_run[0] = s_run[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3285,14 +3444,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_262 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_263 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_7; } } float tot = s_run[0]; @@ -3301,31 +3460,31 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3386,7 +3545,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[7]; float q_r_8 = q_t0_8; - float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, (dk2b_8 - dkt_4[7]) * k_t_8)) + dgf_8; + float qe_7 = q_r_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[7]); + float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[7], k_t_8, cr_7))) + dgf_8; s_run_0[0] = s_run_0[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3396,14 +3560,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_273 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_274 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 6; @@ -3430,7 +3594,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[6]; float q_r_9 = q_t0_9; - float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, (dk2b_9 - dkt_4[6]) * k_t_9)) + dgf_9; + float qe_8 = q_r_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[6]); + float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[6], k_t_9, cr_8))) + dgf_9; s_run_0[0] = s_run_0[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3440,14 +3609,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_9 = t_9 * qrs; if (t_9 < clen) { { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_275 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_276 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 5; @@ -3474,7 +3643,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[5]; float q_r_10 = q_t0_10; - float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, (dk2b_10 - dkt_4[5]) * k_t_10)) + dgf_10; + float qe_9 = q_r_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[5]); + float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[5], k_t_10, cr_9))) + dgf_10; s_run_0[0] = s_run_0[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3484,14 +3658,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_10 = t_10 * qrs; if (t_10 < clen) { { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_277 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_278 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 4; @@ -3518,7 +3692,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[4]; float q_r_11 = q_t0_11; - float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, (dk2b_11 - dkt_4[4]) * k_t_11)) + dgf_11; + float qe_10 = q_r_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[4]); + float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[4], k_t_11, cr_10))) + dgf_11; s_run_0[0] = s_run_0[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3528,14 +3707,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_11 = t_11 * qrs; if (t_11 < clen) { { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_279 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_280 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 3; @@ -3562,7 +3741,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[3]; float q_r_12 = q_t0_12; - float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, (dk2b_12 - dkt_4[3]) * k_t_12)) + dgf_12; + float qe_11 = q_r_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[3]); + float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[3], k_t_12, cr_11))) + dgf_12; s_run_0[0] = s_run_0[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3572,14 +3756,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_12 = t_12 * qrs; if (t_12 < clen) { { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_281 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_282 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 2; @@ -3606,7 +3790,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[2]; float q_r_13 = q_t0_13; - float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, (dk2b_13 - dkt_4[2]) * k_t_13)) + dgf_13; + float qe_12 = q_r_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[2]); + float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[2], k_t_13, cr_12))) + dgf_13; s_run_0[0] = s_run_0[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3616,14 +3805,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_13 = t_13 * qrs; if (t_13 < clen) { { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_283 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_284 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 1; @@ -3650,7 +3839,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[1]; float q_r_14 = q_t0_14; - float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, (dk2b_14 - dkt_4[1]) * k_t_14)) + dgf_14; + float qe_13 = q_r_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[1]); + float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[1], k_t_14, cr_13))) + dgf_14; s_run_0[0] = s_run_0[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3660,14 +3854,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_14 = t_14 * qrs; if (t_14 < clen) { { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_285 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_286 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th; @@ -3694,7 +3888,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[0]; float q_r_15 = q_t0_15; - float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, (dk2b_15 - dkt_4[0]) * k_t_15)) + dgf_15; + float qe_14 = q_r_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[0]); + float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[0], k_t_15, cr_14))) + dgf_15; s_run_0[0] = s_run_0[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3704,14 +3903,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_15 = t_15 * qrs; if (t_15 < clen) { { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_287 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_288 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_15; } } float tot_0 = s_run_0[0]; @@ -3720,36 +3919,36 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_289; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_290; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_291; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_292; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_293; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_294; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3778,9 +3977,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_0 = offset + sfx[0]; if (t_16 < clen) { { - auto _gst_267 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_0; + auto _gst_299 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_0; } } int t_17 = 48 + 8 * th + 1; @@ -3788,9 +3987,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_1 = offset + sfx[1]; if (t_17 < clen) { { - auto _gst_268 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1; + auto _gst_300 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1; } } int t_18 = 48 + 8 * th + 2; @@ -3798,9 +3997,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_2 = offset + sfx[2]; if (t_18 < clen) { { - auto _gst_269 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_2; + auto _gst_301 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_2; } } int t_19 = 48 + 8 * th + 3; @@ -3808,9 +4007,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_3 = offset + sfx[3]; if (t_19 < clen) { { - auto _gst_270 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_3; + auto _gst_302 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_3; } } int t_20 = 48 + 8 * th + 4; @@ -3818,9 +4017,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_4 = offset + sfx[4]; if (t_20 < clen) { { - auto _gst_271 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_4; + auto _gst_303 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_4; } } int t_21 = 48 + 8 * th + 5; @@ -3828,9 +4027,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_5 = offset + sfx[5]; if (t_21 < clen) { { - auto _gst_272 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_5; + auto _gst_304 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_5; } } int t_22 = 48 + 8 * th + 6; @@ -3838,9 +4037,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_6 = offset + sfx[6]; if (t_22 < clen) { { - auto _gst_273 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_6; + auto _gst_305 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_6; } } int t_23 = 48 + 8 * th + 7; @@ -3848,9 +4047,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_7 = offset + sfx[7]; if (t_23 < clen) { { - auto _gst_274 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_7; + auto _gst_306 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3868,9 +4067,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_8 = offset_0 + sfx_0[0]; if (t_24 < clen) { { - auto _gst_275 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_8; + auto _gst_307 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_8; } } int t_25 = 32 + 8 * th + 1; @@ -3878,9 +4077,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_9 = offset_0 + sfx_0[1]; if (t_25 < clen) { { - auto _gst_276 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_9; + auto _gst_308 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_9; } } int t_26 = 32 + 8 * th + 2; @@ -3888,9 +4087,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_10 = offset_0 + sfx_0[2]; if (t_26 < clen) { { - auto _gst_277 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_10; + auto _gst_309 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_10; } } int t_27 = 32 + 8 * th + 3; @@ -3898,9 +4097,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_11 = offset_0 + sfx_0[3]; if (t_27 < clen) { { - auto _gst_278 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_11; + auto _gst_310 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_11; } } int t_28 = 32 + 8 * th + 4; @@ -3908,9 +4107,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_12 = offset_0 + sfx_0[4]; if (t_28 < clen) { { - auto _gst_279 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_12; + auto _gst_311 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_12; } } int t_29 = 32 + 8 * th + 5; @@ -3918,9 +4117,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_13 = offset_0 + sfx_0[5]; if (t_29 < clen) { { - auto _gst_280 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_13; + auto _gst_312 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_13; } } int t_30 = 32 + 8 * th + 6; @@ -3928,9 +4127,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_14 = offset_0 + sfx_0[6]; if (t_30 < clen) { { - auto _gst_281 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_14; + auto _gst_313 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_14; } } int t_31 = 32 + 8 * th + 7; @@ -3938,15 +4137,15 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_15 = offset_0 + sfx_0[7]; if (t_31 < clen) { { - auto _gst_282 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_15; + auto _gst_314 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4007,7 +4206,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[7]; float q_r_16 = q_t0_16; - float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, (dk2b_16 - dkt_5[7]) * k_t_16)) + dgf_16; + float qe_15 = q_r_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[7]); + float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[7], k_t_16, cr_15))) + dgf_16; s_run_1[0] = s_run_1[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4017,14 +4221,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_284 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_316 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_285 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_317 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_33 = 16 + 8 * th + 6; @@ -4051,7 +4255,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[6]; float q_r_17 = q_t0_17; - float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, (dk2b_17 - dkt_5[6]) * k_t_17)) + dgf_17; + float qe_16 = q_r_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[6]); + float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[6], k_t_17, cr_16))) + dgf_17; s_run_1[0] = s_run_1[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4061,14 +4270,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_17 = t_33 * qrs; if (t_33 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_34 = 16 + 8 * th + 5; @@ -4095,7 +4304,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[5]; float q_r_18 = q_t0_18; - float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, (dk2b_18 - dkt_5[5]) * k_t_18)) + dgf_18; + float qe_17 = q_r_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[5]); + float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[5], k_t_18, cr_17))) + dgf_18; s_run_1[0] = s_run_1[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4105,14 +4319,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_18 = t_34 * qrs; if (t_34 < clen) { { - auto _gst_288 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_320 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_289 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_321 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_35 = 16 + 8 * th + 4; @@ -4139,7 +4353,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[4]; float q_r_19 = q_t0_19; - float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, (dk2b_19 - dkt_5[4]) * k_t_19)) + dgf_19; + float qe_18 = q_r_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[4]); + float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[4], k_t_19, cr_18))) + dgf_19; s_run_1[0] = s_run_1[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4149,14 +4368,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_19 = t_35 * qrs; if (t_35 < clen) { { - auto _gst_290 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_322 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_291 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_323 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_36 = 16 + 8 * th + 3; @@ -4183,7 +4402,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[3]; float q_r_20 = q_t0_20; - float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, (dk2b_20 - dkt_5[3]) * k_t_20)) + dgf_20; + float qe_19 = q_r_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[3]); + float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[3], k_t_20, cr_19))) + dgf_20; s_run_1[0] = s_run_1[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4193,14 +4417,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_20 = t_36 * qrs; if (t_36 < clen) { { - auto _gst_292 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_324 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_293 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_325 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_37 = 16 + 8 * th + 2; @@ -4227,7 +4451,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[2]; float q_r_21 = q_t0_21; - float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, (dk2b_21 - dkt_5[2]) * k_t_21)) + dgf_21; + float qe_20 = q_r_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[2]); + float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[2], k_t_21, cr_20))) + dgf_21; s_run_1[0] = s_run_1[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4237,14 +4466,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_21 = t_37 * qrs; if (t_37 < clen) { { - auto _gst_294 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_326 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_295 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_327 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_38 = 16 + 8 * th + 1; @@ -4271,7 +4500,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[1]; float q_r_22 = q_t0_22; - float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, (dk2b_22 - dkt_5[1]) * k_t_22)) + dgf_22; + float qe_21 = q_r_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[1]); + float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[1], k_t_22, cr_21))) + dgf_22; s_run_1[0] = s_run_1[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4281,14 +4515,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_22 = t_38 * qrs; if (t_38 < clen) { { - auto _gst_296 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_328 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_297 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_329 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_39 = 16 + 8 * th; @@ -4315,7 +4549,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[0]; float q_r_23 = q_t0_23; - float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, (dk2b_23 - dkt_5[0]) * k_t_23)) + dgf_23; + float qe_22 = q_r_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[0]); + float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[0], k_t_23, cr_22))) + dgf_23; s_run_1[0] = s_run_1[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4325,14 +4564,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_23 = t_39 * qrs; if (t_39 < clen) { { - auto _gst_298 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_330 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_299 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_331 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_23; } } float tot_1 = s_run_1[0]; @@ -4341,37 +4580,37 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_332; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_333; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_334; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_335; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_336; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_337; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4400,9 +4639,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_16 = offset_1 + sfx_1[0]; if (t_40 < clen) { { - auto _gst_310 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_16; + auto _gst_342 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_16; } } int t_41 = 16 + 8 * th + 1; @@ -4410,9 +4649,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_17 = offset_1 + sfx_1[1]; if (t_41 < clen) { { - auto _gst_311 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_17; + auto _gst_343 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_17; } } int t_42 = 16 + 8 * th + 2; @@ -4420,9 +4659,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_18 = offset_1 + sfx_1[2]; if (t_42 < clen) { { - auto _gst_312 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_18; + auto _gst_344 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_18; } } int t_43 = 16 + 8 * th + 3; @@ -4430,9 +4669,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_19 = offset_1 + sfx_1[3]; if (t_43 < clen) { { - auto _gst_313 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_19; + auto _gst_345 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_19; } } int t_44 = 16 + 8 * th + 4; @@ -4440,9 +4679,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_20 = offset_1 + sfx_1[4]; if (t_44 < clen) { { - auto _gst_314 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_20; + auto _gst_346 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_20; } } int t_45 = 16 + 8 * th + 5; @@ -4450,9 +4689,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_21 = offset_1 + sfx_1[5]; if (t_45 < clen) { { - auto _gst_315 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_21; + auto _gst_347 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_21; } } int t_46 = 16 + 8 * th + 6; @@ -4460,9 +4699,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_22 = offset_1 + sfx_1[6]; if (t_46 < clen) { { - auto _gst_316 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_22; + auto _gst_348 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_22; } } int t_47 = 16 + 8 * th + 7; @@ -4470,15 +4709,15 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_23 = offset_1 + sfx_1[7]; if (t_47 < clen) { { - auto _gst_317 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_23; + auto _gst_349 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4533,7 +4772,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[7]; float q_r_24 = q_t0_24; - float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, (dk2b_24 - dkt_6[7]) * k_t_24)) + dgf_24; + float qe_23 = q_r_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[7]); + float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[7], k_t_24, cr_23))) + dgf_24; s_run_2[0] = s_run_2[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4543,14 +4787,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_319 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_351 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_320 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_352 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_49 = 8 * th + 6; @@ -4577,7 +4821,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[6]; float q_r_25 = q_t0_25; - float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, (dk2b_25 - dkt_6[6]) * k_t_25)) + dgf_25; + float qe_24 = q_r_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[6]); + float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[6], k_t_25, cr_24))) + dgf_25; s_run_2[0] = s_run_2[0] + dg_o_25; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4587,14 +4836,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_25 = t_49 * qrs; if (t_49 < clen) { { - auto _gst_321 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_353 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_322 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_354 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_50 = 8 * th + 5; @@ -4621,7 +4870,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[5]; float q_r_26 = q_t0_26; - float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, (dk2b_26 - dkt_6[5]) * k_t_26)) + dgf_26; + float qe_25 = q_r_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[5]); + float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[5], k_t_26, cr_25))) + dgf_26; s_run_2[0] = s_run_2[0] + dg_o_26; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4631,14 +4885,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_26 = t_50 * qrs; if (t_50 < clen) { { - auto _gst_323 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_355 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_324 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_356 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_51 = 8 * th + 4; @@ -4665,7 +4919,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[4]; float q_r_27 = q_t0_27; - float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, (dk2b_27 - dkt_6[4]) * k_t_27)) + dgf_27; + float qe_26 = q_r_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[4]); + float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[4], k_t_27, cr_26))) + dgf_27; s_run_2[0] = s_run_2[0] + dg_o_27; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4675,14 +4934,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_27 = t_51 * qrs; if (t_51 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_52 = 8 * th + 3; @@ -4709,7 +4968,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[3]; float q_r_28 = q_t0_28; - float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, (dk2b_28 - dkt_6[3]) * k_t_28)) + dgf_28; + float qe_27 = q_r_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[3]); + float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[3], k_t_28, cr_27))) + dgf_28; s_run_2[0] = s_run_2[0] + dg_o_28; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4719,14 +4983,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_28 = t_52 * qrs; if (t_52 < clen) { { - auto _gst_327 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_359 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_328 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_360 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_53 = 8 * th + 2; @@ -4753,7 +5017,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[2]; float q_r_29 = q_t0_29; - float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, (dk2b_29 - dkt_6[2]) * k_t_29)) + dgf_29; + float qe_28 = q_r_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[2]); + float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[2], k_t_29, cr_28))) + dgf_29; s_run_2[0] = s_run_2[0] + dg_o_29; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4763,14 +5032,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_29 = t_53 * qrs; if (t_53 < clen) { { - auto _gst_329 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_361 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_330 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_362 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_54 = 8 * th + 1; @@ -4797,7 +5066,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[1]; float q_r_30 = q_t0_30; - float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, (dk2b_30 - dkt_6[1]) * k_t_30)) + dgf_30; + float qe_29 = q_r_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[1]); + float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[1], k_t_30, cr_29))) + dgf_30; s_run_2[0] = s_run_2[0] + dg_o_30; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4807,14 +5081,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_30 = t_54 * qrs; if (t_54 < clen) { { - auto _gst_331 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_363 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_332 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_364 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_55 = 8 * th; @@ -4841,7 +5115,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[0]; float q_r_31 = q_t0_31; - float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, (dk2b_31 - dkt_6[0]) * k_t_31)) + dgf_31; + float qe_30 = q_r_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[0]); + float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[0], k_t_31, cr_30))) + dgf_31; s_run_2[0] = s_run_2[0] + dg_o_31; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4851,14 +5130,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_31 = t_55 * qrs; if (t_55 < clen) { { - auto _gst_333 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_365 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_334 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_366 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_31; } } float tot_2 = s_run_2[0]; @@ -4867,35 +5146,35 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_367; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_368; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_369; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_370; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_371; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_372; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_2) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4924,9 +5203,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_24 = offset_2 + sfx_2[0]; if (t_56 < clen) { { - auto _gst_345 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_24; + auto _gst_377 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_24; } } int t_57 = 8 * th + 1; @@ -4934,9 +5213,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_25 = offset_2 + sfx_2[1]; if (t_57 < clen) { { - auto _gst_346 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_25; + auto _gst_378 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_25; } } int t_58 = 8 * th + 2; @@ -4944,9 +5223,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_26 = offset_2 + sfx_2[2]; if (t_58 < clen) { { - auto _gst_347 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_26; + auto _gst_379 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_26; } } int t_59 = 8 * th + 3; @@ -4954,9 +5233,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_27 = offset_2 + sfx_2[3]; if (t_59 < clen) { { - auto _gst_348 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_27; + auto _gst_380 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_27; } } int t_60 = 8 * th + 4; @@ -4964,9 +5243,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_28 = offset_2 + sfx_2[4]; if (t_60 < clen) { { - auto _gst_349 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_28; + auto _gst_381 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_28; } } int t_61 = 8 * th + 5; @@ -4974,9 +5253,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_29 = offset_2 + sfx_2[5]; if (t_61 < clen) { { - auto _gst_350 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_29; + auto _gst_382 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_29; } } int t_62 = 8 * th + 6; @@ -4984,9 +5263,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_30 = offset_2 + sfx_2[6]; if (t_62 < clen) { { - auto _gst_351 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_30; + auto _gst_383 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_30; } } int t_63 = 8 * th + 7; @@ -4994,9 +5273,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_31 = offset_2 + sfx_2[7]; if (t_63 < clen) { { - auto _gst_352 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_31; + auto _gst_384 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_31; } } off_acc[0] = off_acc[0] + t_hi_2 + t_lo_2; diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu index 952c3625eee..9970f4baf13 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2249,12 +2272,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_3 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * e2_3); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_3, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_3 : __fmaf_rn(dl, _rcp_170, e2_3)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2278,12 +2307,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_4 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * e2_4); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_4, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_4, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_4 : __fmaf_rn(dl_1, _rcp_174, e2_4)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2307,12 +2342,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_5 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * e2_5); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_5, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_5, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_5 : __fmaf_rn(dl_3, _rcp_178, e2_5)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2336,12 +2377,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_6 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * e2_6); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_6, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_6, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_6 : __fmaf_rn(dl_5, _rcp_182, e2_6)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_2 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_1 = approx_exp2(dc2_2.x); float e2_1_1 = approx_exp2(dc2_2.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_190 = approx_rcp(k_t0_2); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_191 = approx_rcp(k_t1_2); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_2.x; pan_641[3] = p2_2.y; int t0_3 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_2 = approx_exp2(dc2_3.x); float e2_1_2 = approx_exp2(dc2_3.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_194 = approx_rcp(k_t0_3); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_195 = approx_rcp(k_t1_3); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_3.x; pan_641[5] = p2_3.y; int t0_4 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(_f2_61, make_float2(g_t0_4, g_t1_4)); float e2_0_3 = approx_exp2(dc2_4.x); float e2_1_3 = approx_exp2(dc2_4.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_198 = approx_rcp(k_t0_4); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_199 = approx_rcp(k_t1_4); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_4.x; pan_641[7] = p2_4.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2505,12 +2576,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_202 = approx_rcp(k_t0_5); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_203 = approx_rcp(k_t1_5); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_5.x; pan_703[1] = p2_5.y; int t0_6 = 8 * th + 2; @@ -2534,12 +2611,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_2 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_2.x); float en_1_1 = approx_exp2(dn2_2.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_206 = approx_rcp(k_t0_6); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_207 = approx_rcp(k_t1_6); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_6.x; pan_703[3] = p2_6.y; int t0_7 = 8 * th + 4; @@ -2563,12 +2646,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_3 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_3.x); float en_1_2 = approx_exp2(dn2_3.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_210 = approx_rcp(k_t0_7); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_211 = approx_rcp(k_t1_7); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_7.x; pan_703[5] = p2_7.y; int t0_8 = 8 * th + 6; @@ -2592,12 +2681,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_4 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_4.x); float en_1_3 = approx_exp2(dn2_4.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_214 = approx_rcp(k_t0_8); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_215 = approx_rcp(k_t1_8); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_8.x; pan_703[7] = p2_8.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2639,12 +2734,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_5 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_5.x); float en_1_4 = approx_exp2(dn2_5.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_218 = approx_rcp(k_t0_9); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_219 = approx_rcp(k_t1_9); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_9.x; pan_777[1] = p2_9.y; int t0_10 = 16 + 8 * th + 2; @@ -2668,12 +2769,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_6 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_6.x); float en_1_5 = approx_exp2(dn2_6.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_222 = approx_rcp(k_t0_10); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_223 = approx_rcp(k_t1_10); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_10.x; pan_777[3] = p2_10.y; int t0_11 = 16 + 8 * th + 4; @@ -2697,12 +2804,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_7 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_7.x); float en_1_6 = approx_exp2(dn2_7.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_226 = approx_rcp(k_t0_11); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_227 = approx_rcp(k_t1_11); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_11.x; pan_777[5] = p2_11.y; int t0_12 = 16 + 8 * th + 6; @@ -2726,12 +2839,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_8 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_8.x); float en_1_7 = approx_exp2(dn2_8.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_230 = approx_rcp(k_t0_12); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_231 = approx_rcp(k_t1_12); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_12.x; pan_777[7] = p2_12.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2802,37 +2921,37 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_4 = approx_exp2(d2_57.x); float e1_4 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_4, e1_4); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_4, e1_4); float2 v2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_4.x; v_562_0[1] = v2_4.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_5 = approx_exp2(d2_58.x); float e1_5 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_5, e1_5); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_5, e1_5); float2 v2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_5.x; v_562_0[3] = v2_5.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_6 = approx_exp2(d2_59.x); float e1_6 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_6, e1_6); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_6, e1_6); float2 v2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_6.x; v_562_0[5] = v2_6.y; float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_7 = approx_exp2(d2_60.x); float e1_7 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_7, e1_7); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_7, e1_7); float2 v2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_7.x; v_562_0[7] = v2_7.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2966,7 +3085,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -2976,14 +3100,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_0 = t_1_0 * qrs; if (t_1_0 < clen) { { - auto _gst_216 = dq_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_1_0; + auto _gst_248 = dq_base + qoff_1_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_1_0; } { - auto _gst_217 = dk_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_1_0; + auto _gst_249 = dk_base + qoff_1_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_1_0; } } int t_1_1 = 32 + 8 * th + 6; @@ -3009,7 +3133,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3019,14 +3148,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_1 = t_1_1 * qrs; if (t_1_1 < clen) { { - auto _gst_218 = dq_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1_1; + auto _gst_250 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1_1; } { - auto _gst_219 = dk_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1_1; + auto _gst_251 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1_1; } } int t_1_2 = 32 + 8 * th + 5; @@ -3052,7 +3181,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_2)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3062,14 +3196,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_2 = t_1_2 * qrs; if (t_1_2 < clen) { { - auto _gst_220 = dq_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1_2; + auto _gst_252 = dq_base + qoff_1_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_1_2; } { - auto _gst_221 = dk_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1_2; + auto _gst_253 = dk_base + qoff_1_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_1_2; } } int t_1_3 = 32 + 8 * th + 4; @@ -3095,7 +3229,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_3)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3105,14 +3244,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_3 = t_1_3 * qrs; if (t_1_3 < clen) { { - auto _gst_222 = dq_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_1_3; + auto _gst_254 = dq_base + qoff_1_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_1_3; } { - auto _gst_223 = dk_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_1_3; + auto _gst_255 = dk_base + qoff_1_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_1_3; } } int t_1_4 = 32 + 8 * th + 3; @@ -3138,7 +3277,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_4)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3148,14 +3292,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_4 = t_1_4 * qrs; if (t_1_4 < clen) { { - auto _gst_224 = dq_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_1_4; + auto _gst_256 = dq_base + qoff_1_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_1_4; } { - auto _gst_225 = dk_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_1_4; + auto _gst_257 = dk_base + qoff_1_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_1_4; } } int t_1_5 = 32 + 8 * th + 2; @@ -3181,7 +3325,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_5)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3191,14 +3340,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_5 = t_1_5 * qrs; if (t_1_5 < clen) { { - auto _gst_226 = dq_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_1_5; + auto _gst_258 = dq_base + qoff_1_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_1_5; } { - auto _gst_227 = dk_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_1_5; + auto _gst_259 = dk_base + qoff_1_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_1_5; } } int t_1_6 = 32 + 8 * th + 1; @@ -3224,7 +3373,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_6)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3234,14 +3388,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_6 = t_1_6 * qrs; if (t_1_6 < clen) { { - auto _gst_228 = dq_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_1_6; + auto _gst_260 = dq_base + qoff_1_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_1_6; } { - auto _gst_229 = dk_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_1_6; + auto _gst_261 = dk_base + qoff_1_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_1_6; } } int t_1_7 = 32 + 8 * th; @@ -3267,7 +3421,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_7)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3277,14 +3436,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_7 = t_1_7 * qrs; if (t_1_7 < clen) { { - auto _gst_230 = dq_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_1_7; + auto _gst_262 = dq_base + qoff_1_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_1_7; } { - auto _gst_231 = dk_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_1_7; + auto _gst_263 = dk_base + qoff_1_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_1_7; } } float tot = s_run[0]; @@ -3293,31 +3452,31 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3377,7 +3536,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; + float qe_7 = q_t_1_8 * e_dg_0; + float kp_19 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_0; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_14[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_0) * _tmem_load_15[7]); + float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_16[7], k_t_0, cr_8)) + dgf_0; s_run_1059[0] = s_run_1059[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3387,14 +3551,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_241 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_273 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_242 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_274 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_2 = 48 + 8 * th + 6; @@ -3420,7 +3584,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[6]; - float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, (dk2b_2 - _tmem_load_16[6]) * k_t_2) + dgf_2; + float qe_8 = q_t_1_9 * e_dg_2; + float kp_20 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_14[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_2) * _tmem_load_15[6]); + float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_16[6], k_t_2, cr_9)) + dgf_2; s_run_1059[0] = s_run_1059[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3430,14 +3599,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_243 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_275 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_244 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_276 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 48 + 8 * th + 5; @@ -3463,7 +3632,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[5]; - float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, (dk2b_3 - _tmem_load_16[5]) * k_t_3) + dgf_3; + float qe_9 = q_t_1_10 * e_dg_3; + float kp_21 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_3; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_14[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_3) * _tmem_load_15[5]); + float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_16[5], k_t_3, cr_10)) + dgf_3; s_run_1059[0] = s_run_1059[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3473,14 +3647,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_245 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_277 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_246 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_278 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 48 + 8 * th + 4; @@ -3506,7 +3680,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[4]; - float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, (dk2b_4 - _tmem_load_16[4]) * k_t_4) + dgf_4; + float qe_10 = q_t_1_11 * e_dg_4; + float kp_22 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_4; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_14[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_4) * _tmem_load_15[4]); + float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_16[4], k_t_4, cr_11)) + dgf_4; s_run_1059[0] = s_run_1059[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3516,14 +3695,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_247 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_279 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_248 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_280 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 48 + 8 * th + 3; @@ -3549,7 +3728,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[3]; - float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, (dk2b_5 - _tmem_load_16[3]) * k_t_5) + dgf_5; + float qe_11 = q_t_1_12 * e_dg_5; + float kp_23 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_5; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_14[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_5) * _tmem_load_15[3]); + float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_16[3], k_t_5, cr_12)) + dgf_5; s_run_1059[0] = s_run_1059[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3559,14 +3743,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_249 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_281 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_250 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_282 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 48 + 8 * th + 2; @@ -3592,7 +3776,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[2]; - float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, (dk2b_6 - _tmem_load_16[2]) * k_t_6) + dgf_6; + float qe_12 = q_t_1_13 * e_dg_6; + float kp_24 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_6; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_14[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_6) * _tmem_load_15[2]); + float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_16[2], k_t_6, cr_13)) + dgf_6; s_run_1059[0] = s_run_1059[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3602,14 +3791,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_251 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_283 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_252 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_284 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 48 + 8 * th + 1; @@ -3635,7 +3824,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[1]; - float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, (dk2b_7 - _tmem_load_16[1]) * k_t_7) + dgf_7; + float qe_13 = q_t_1_14 * e_dg_7; + float kp_25 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_7; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_14[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_7) * _tmem_load_15[1]); + float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_16[1], k_t_7, cr_14)) + dgf_7; s_run_1059[0] = s_run_1059[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3645,14 +3839,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_253 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_285 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_254 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_286 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_7; } } int t_8 = 48 + 8 * th; @@ -3678,7 +3872,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_16[0]; - float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, (dk2b_8 - _tmem_load_16[0]) * k_t_8) + dgf_8; + float qe_14 = q_t_1_15 * e_dg_8; + float kp_26 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_8; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_14[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_8) * _tmem_load_15[0]); + float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_16[0], k_t_8, cr_15)) + dgf_8; s_run_1059[0] = s_run_1059[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3688,14 +3887,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_255 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_287 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_256 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_288 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_8; } } float tot_1228 = s_run_1059[0]; @@ -3704,36 +3903,36 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int b2_1231 = lane_1 >> 2 & 1; float v4_1232[4]; float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_289; float send_2 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_290; float send_3 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_291; float send_4 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_292; float v2_1237[2]; float send_5 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_293; float send_6 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); + v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_294; float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); + float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); + v1_1241 = v1_1241 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); + v1_1241 = v1_1241 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_1228) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_1228) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3762,9 +3961,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_0 = offset + _tmem_load_17[0]; if (t_9 < clen) { { - auto _gst_267 = dg_base + goff_1_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_1_0; + auto _gst_299 = dg_base + goff_1_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_1_0; } } int t_10 = 48 + 8 * th + 1; @@ -3772,9 +3971,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_1 = offset + _tmem_load_17[1]; if (t_10 < clen) { { - auto _gst_268 = dg_base + goff_1_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1_1; + auto _gst_300 = dg_base + goff_1_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1_1; } } int t_11 = 48 + 8 * th + 2; @@ -3782,9 +3981,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_2 = offset + _tmem_load_17[2]; if (t_11 < clen) { { - auto _gst_269 = dg_base + goff_1_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_1_2; + auto _gst_301 = dg_base + goff_1_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_1_2; } } int t_12 = 48 + 8 * th + 3; @@ -3792,9 +3991,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_3 = offset + _tmem_load_17[3]; if (t_12 < clen) { { - auto _gst_270 = dg_base + goff_1_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_1_3; + auto _gst_302 = dg_base + goff_1_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_1_3; } } int t_13 = 48 + 8 * th + 4; @@ -3802,9 +4001,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_4 = offset + _tmem_load_17[4]; if (t_13 < clen) { { - auto _gst_271 = dg_base + goff_1_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_1_4; + auto _gst_303 = dg_base + goff_1_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_1_4; } } int t_14 = 48 + 8 * th + 5; @@ -3812,9 +4011,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_5 = offset + _tmem_load_17[5]; if (t_14 < clen) { { - auto _gst_272 = dg_base + goff_1_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_1_5; + auto _gst_304 = dg_base + goff_1_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_1_5; } } int t_15 = 48 + 8 * th + 6; @@ -3822,9 +4021,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_6 = offset + _tmem_load_17[6]; if (t_15 < clen) { { - auto _gst_273 = dg_base + goff_1_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_1_6; + auto _gst_305 = dg_base + goff_1_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_1_6; } } int t_16 = 48 + 8 * th + 7; @@ -3832,9 +4031,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_7 = offset + _tmem_load_17[7]; if (t_16 < clen) { { - auto _gst_274 = dg_base + goff_1_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_1_7; + auto _gst_306 = dg_base + goff_1_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_1_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3852,9 +4051,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_0 = offset_1250 + _tmem_load_19[0]; if (t_17 < clen) { { - auto _gst_275 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_0; + auto _gst_307 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_0; } } int t_18 = 32 + 8 * th + 1; @@ -3862,9 +4061,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1 = offset_1250 + _tmem_load_19[1]; if (t_18 < clen) { { - auto _gst_276 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_1; + auto _gst_308 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_1; } } int t_19 = 32 + 8 * th + 2; @@ -3872,9 +4071,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_2 = offset_1250 + _tmem_load_19[2]; if (t_19 < clen) { { - auto _gst_277 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_2; + auto _gst_309 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_2; } } int t_20 = 32 + 8 * th + 3; @@ -3882,9 +4081,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_3 = offset_1250 + _tmem_load_19[3]; if (t_20 < clen) { { - auto _gst_278 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_3; + auto _gst_310 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_3; } } int t_21 = 32 + 8 * th + 4; @@ -3892,9 +4091,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_4 = offset_1250 + _tmem_load_19[4]; if (t_21 < clen) { { - auto _gst_279 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_4; + auto _gst_311 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_4; } } int t_22 = 32 + 8 * th + 5; @@ -3902,9 +4101,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_5 = offset_1250 + _tmem_load_19[5]; if (t_22 < clen) { { - auto _gst_280 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_5; + auto _gst_312 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_5; } } int t_23 = 32 + 8 * th + 6; @@ -3912,9 +4111,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_6 = offset_1250 + _tmem_load_19[6]; if (t_23 < clen) { { - auto _gst_281 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_6; + auto _gst_313 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_6; } } int t_24 = 32 + 8 * th + 7; @@ -3922,15 +4121,15 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_7 = offset_1250 + _tmem_load_19[7]; if (t_24 < clen) { { - auto _gst_282 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_7; + auto _gst_314 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi_1248 + t_lo_1249; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3990,7 +4189,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[7]; - float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, (dk2b_9 - _tmem_load_24[7]) * k_t_9) + dgf_9; + float qe_15 = q_t_1_16 * e_dg_9; + float kp_27 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_9; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_9) * _tmem_load_23[7]); + float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_24[7], k_t_9, cr_16)) + dgf_9; s_run_1302[0] = s_run_1302[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4000,14 +4204,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_9 = t_25 * qrs; if (t_25 < clen) { { - auto _gst_284 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_316 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_285 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_317 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_26 = 16 + 8 * th + 6; @@ -4033,7 +4237,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[6]; - float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, (dk2b_10 - _tmem_load_24[6]) * k_t_10) + dgf_10; + float qe_16 = q_t_1_17 * e_dg_10; + float kp_28 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_10; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_10) * _tmem_load_23[6]); + float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_24[6], k_t_10, cr_17)) + dgf_10; s_run_1302[0] = s_run_1302[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4043,14 +4252,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_10 = t_26 * qrs; if (t_26 < clen) { { - auto _gst_286 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_318 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_287 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_319 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_27 = 16 + 8 * th + 5; @@ -4076,7 +4285,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[5]; - float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, (dk2b_11 - _tmem_load_24[5]) * k_t_11) + dgf_11; + float qe_17 = q_t_1_18 * e_dg_11; + float kp_29 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_11; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_11) * _tmem_load_23[5]); + float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_24[5], k_t_11, cr_18)) + dgf_11; s_run_1302[0] = s_run_1302[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4086,14 +4300,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_11 = t_27 * qrs; if (t_27 < clen) { { - auto _gst_288 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_320 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_289 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_321 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_28 = 16 + 8 * th + 4; @@ -4119,7 +4333,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[4]; - float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, (dk2b_12 - _tmem_load_24[4]) * k_t_12) + dgf_12; + float qe_18 = q_t_1_19 * e_dg_12; + float kp_30 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_12; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_12) * _tmem_load_23[4]); + float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_24[4], k_t_12, cr_19)) + dgf_12; s_run_1302[0] = s_run_1302[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4129,14 +4348,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_12 = t_28 * qrs; if (t_28 < clen) { { - auto _gst_290 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_322 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_291 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_323 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_29 = 16 + 8 * th + 3; @@ -4162,7 +4381,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[3]; - float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, (dk2b_13 - _tmem_load_24[3]) * k_t_13) + dgf_13; + float qe_19 = q_t_1_20 * e_dg_13; + float kp_31 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_13; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_13) * _tmem_load_23[3]); + float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_24[3], k_t_13, cr_20)) + dgf_13; s_run_1302[0] = s_run_1302[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4172,14 +4396,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_13 = t_29 * qrs; if (t_29 < clen) { { - auto _gst_292 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_324 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_293 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_325 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_30 = 16 + 8 * th + 2; @@ -4205,7 +4429,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[2]; - float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, (dk2b_14 - _tmem_load_24[2]) * k_t_14) + dgf_14; + float qe_20 = q_t_1_21 * e_dg_14; + float kp_32 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_14; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_14) * _tmem_load_23[2]); + float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_24[2], k_t_14, cr_21)) + dgf_14; s_run_1302[0] = s_run_1302[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4215,14 +4444,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_14 = t_30 * qrs; if (t_30 < clen) { { - auto _gst_294 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_326 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_295 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_327 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_31 = 16 + 8 * th + 1; @@ -4248,7 +4477,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[1]; - float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, (dk2b_15 - _tmem_load_24[1]) * k_t_15) + dgf_15; + float qe_21 = q_t_1_22 * e_dg_15; + float kp_33 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_15; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_15) * _tmem_load_23[1]); + float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_24[1], k_t_15, cr_22)) + dgf_15; s_run_1302[0] = s_run_1302[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4258,14 +4492,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_15 = t_31 * qrs; if (t_31 < clen) { { - auto _gst_296 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_328 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_297 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_329 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_15; } } int t_32 = 16 + 8 * th; @@ -4291,7 +4525,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_24[0]; - float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, (dk2b_16 - _tmem_load_24[0]) * k_t_16) + dgf_16; + float qe_22 = q_t_1_23 * e_dg_16; + float kp_34 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_16; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_16) * _tmem_load_23[0]); + float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, __fmaf_rn(dk2b_16 - _tmem_load_24[0], k_t_16, cr_23)) + dgf_16; s_run_1302[0] = s_run_1302[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4301,14 +4540,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_298 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_330 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_299 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_331 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_16; } } float tot_1471 = s_run_1302[0]; @@ -4317,37 +4556,37 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; float send_7 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_332; float send_8 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_333; float send_9 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_334; float send_10 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_335; float v2_1480[2]; float send_11 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_336; float send_12 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_337; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1471) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4376,9 +4615,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_8 = offset_1488 + _tmem_load_25[0]; if (t_33 < clen) { { - auto _gst_310 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_8; + auto _gst_342 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_8; } } int t_34 = 16 + 8 * th + 1; @@ -4386,9 +4625,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_9 = offset_1488 + _tmem_load_25[1]; if (t_34 < clen) { { - auto _gst_311 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_9; + auto _gst_343 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_9; } } int t_35 = 16 + 8 * th + 2; @@ -4396,9 +4635,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_10 = offset_1488 + _tmem_load_25[2]; if (t_35 < clen) { { - auto _gst_312 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_10; + auto _gst_344 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_10; } } int t_36 = 16 + 8 * th + 3; @@ -4406,9 +4645,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_11 = offset_1488 + _tmem_load_25[3]; if (t_36 < clen) { { - auto _gst_313 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_11; + auto _gst_345 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_11; } } int t_37 = 16 + 8 * th + 4; @@ -4416,9 +4655,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_12 = offset_1488 + _tmem_load_25[4]; if (t_37 < clen) { { - auto _gst_314 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_12; + auto _gst_346 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_12; } } int t_38 = 16 + 8 * th + 5; @@ -4426,9 +4665,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_13 = offset_1488 + _tmem_load_25[5]; if (t_38 < clen) { { - auto _gst_315 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_13; + auto _gst_347 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_13; } } int t_39 = 16 + 8 * th + 6; @@ -4436,9 +4675,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_14 = offset_1488 + _tmem_load_25[6]; if (t_39 < clen) { { - auto _gst_316 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_14; + auto _gst_348 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_14; } } int t_40 = 16 + 8 * th + 7; @@ -4446,15 +4685,15 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_15 = offset_1488 + _tmem_load_25[7]; if (t_40 < clen) { { - auto _gst_317 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_15; + auto _gst_349 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_1486 + t_lo_1487; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4508,7 +4747,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[7]; - float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, (dk2b_17 - _tmem_load_28[7]) * k_t_17) + dgf_17; + float qe_23 = q_t_1_24 * e_dg_17; + float kp_35 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_35; + float kbe_23 = (float)kb_23 * e_dg_17; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_35 * e_dg_17) * _tmem_load_27[7]); + float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, __fmaf_rn(dk2b_17 - _tmem_load_28[7], k_t_17, cr_24)) + dgf_17; s_run_1524[0] = s_run_1524[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4518,14 +4762,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_17 = t_41 * qrs; if (t_41 < clen) { { - auto _gst_319 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_351 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_320 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_352 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_42 = 8 * th + 6; @@ -4551,7 +4795,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[6]; - float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, (dk2b_18 - _tmem_load_28[6]) * k_t_18) + dgf_18; + float qe_24 = q_t_1_25 * e_dg_18; + float kp_36 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_36; + float kbe_24 = (float)kb_24 * e_dg_18; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_36 * e_dg_18) * _tmem_load_27[6]); + float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, __fmaf_rn(dk2b_18 - _tmem_load_28[6], k_t_18, cr_25)) + dgf_18; s_run_1524[0] = s_run_1524[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4561,14 +4810,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_18 = t_42 * qrs; if (t_42 < clen) { { - auto _gst_321 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_353 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_322 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_354 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_43 = 8 * th + 5; @@ -4594,7 +4843,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[5]; - float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, (dk2b_19 - _tmem_load_28[5]) * k_t_19) + dgf_19; + float qe_25 = q_t_1_26 * e_dg_19; + float kp_37 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_37; + float kbe_25 = (float)kb_25 * e_dg_19; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_37 * e_dg_19) * _tmem_load_27[5]); + float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, __fmaf_rn(dk2b_19 - _tmem_load_28[5], k_t_19, cr_26)) + dgf_19; s_run_1524[0] = s_run_1524[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4604,14 +4858,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_19 = t_43 * qrs; if (t_43 < clen) { { - auto _gst_323 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_355 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_324 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_356 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_44 = 8 * th + 4; @@ -4637,7 +4891,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[4]; - float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, (dk2b_20 - _tmem_load_28[4]) * k_t_20) + dgf_20; + float qe_26 = q_t_1_27 * e_dg_20; + float kp_38 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_38; + float kbe_26 = (float)kb_26 * e_dg_20; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_38 * e_dg_20) * _tmem_load_27[4]); + float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, __fmaf_rn(dk2b_20 - _tmem_load_28[4], k_t_20, cr_27)) + dgf_20; s_run_1524[0] = s_run_1524[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4647,14 +4906,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_20 = t_44 * qrs; if (t_44 < clen) { { - auto _gst_325 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_357 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_326 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_358 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_45 = 8 * th + 3; @@ -4680,7 +4939,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[3]; - float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, (dk2b_21 - _tmem_load_28[3]) * k_t_21) + dgf_21; + float qe_27 = q_t_1_28 * e_dg_21; + float kp_39 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_39; + float kbe_27 = (float)kb_27 * e_dg_21; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_39 * e_dg_21) * _tmem_load_27[3]); + float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, __fmaf_rn(dk2b_21 - _tmem_load_28[3], k_t_21, cr_28)) + dgf_21; s_run_1524[0] = s_run_1524[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4690,14 +4954,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_21 = t_45 * qrs; if (t_45 < clen) { { - auto _gst_327 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_359 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_328 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_360 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_46 = 8 * th + 2; @@ -4723,7 +4987,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[2]; - float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, (dk2b_22 - _tmem_load_28[2]) * k_t_22) + dgf_22; + float qe_28 = q_t_1_29 * e_dg_22; + float kp_40 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_40; + float kbe_28 = (float)kb_28 * e_dg_22; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_40 * e_dg_22) * _tmem_load_27[2]); + float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, __fmaf_rn(dk2b_22 - _tmem_load_28[2], k_t_22, cr_29)) + dgf_22; s_run_1524[0] = s_run_1524[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4733,14 +5002,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_22 = t_46 * qrs; if (t_46 < clen) { { - auto _gst_329 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_361 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_330 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_362 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_47 = 8 * th + 1; @@ -4766,7 +5035,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[1]; - float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, (dk2b_23 - _tmem_load_28[1]) * k_t_23) + dgf_23; + float qe_29 = q_t_1_30 * e_dg_23; + float kp_41 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_41; + float kbe_29 = (float)kb_29 * e_dg_23; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_41 * e_dg_23) * _tmem_load_27[1]); + float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, __fmaf_rn(dk2b_23 - _tmem_load_28[1], k_t_23, cr_30)) + dgf_23; s_run_1524[0] = s_run_1524[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4776,14 +5050,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_23 = t_47 * qrs; if (t_47 < clen) { { - auto _gst_331 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_363 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_332 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_364 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_23; } } int t_48 = 8 * th; @@ -4809,7 +5083,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + _tmem_load_28[0]; - float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, (dk2b_24 - _tmem_load_28[0]) * k_t_24) + dgf_24; + float qe_30 = q_t_1_31 * e_dg_24; + float kp_42 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_42; + float kbe_30 = (float)kb_30 * e_dg_24; + float cr_31 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_42 * e_dg_24) * _tmem_load_27[0]); + float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, __fmaf_rn(dk2b_24 - _tmem_load_28[0], k_t_24, cr_31)) + dgf_24; s_run_1524[0] = s_run_1524[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4819,14 +5098,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_333 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_365 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_334 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_366 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_24; } } float tot_1693 = s_run_1524[0]; @@ -4835,35 +5114,35 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; float send_13 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_367; float send_14 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_368; float send_15 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_369; float send_16 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_370; float v2_1702[2]; float send_17 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_371; float send_18 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_372; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_1693) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4892,9 +5171,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_16 = offset_1710 + _tmem_load_29[0]; if (t_49 < clen) { { - auto _gst_345 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_16; + auto _gst_377 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_16; } } int t_50 = 8 * th + 1; @@ -4902,9 +5181,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_17 = offset_1710 + _tmem_load_29[1]; if (t_50 < clen) { { - auto _gst_346 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_17; + auto _gst_378 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_17; } } int t_51 = 8 * th + 2; @@ -4912,9 +5191,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_18 = offset_1710 + _tmem_load_29[2]; if (t_51 < clen) { { - auto _gst_347 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_18; + auto _gst_379 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_18; } } int t_52 = 8 * th + 3; @@ -4922,9 +5201,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_19 = offset_1710 + _tmem_load_29[3]; if (t_52 < clen) { { - auto _gst_348 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_19; + auto _gst_380 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_19; } } int t_53 = 8 * th + 4; @@ -4932,9 +5211,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_20 = offset_1710 + _tmem_load_29[4]; if (t_53 < clen) { { - auto _gst_349 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_20; + auto _gst_381 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_20; } } int t_54 = 8 * th + 5; @@ -4942,9 +5221,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_21 = offset_1710 + _tmem_load_29[5]; if (t_54 < clen) { { - auto _gst_350 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_21; + auto _gst_382 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_21; } } int t_55 = 8 * th + 6; @@ -4952,9 +5231,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_22 = offset_1710 + _tmem_load_29[6]; if (t_55 < clen) { { - auto _gst_351 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_22; + auto _gst_383 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_22; } } int t_56 = 8 * th + 7; @@ -4962,9 +5241,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_23 = offset_1710 + _tmem_load_29[7]; if (t_56 < clen) { { - auto _gst_352 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_23; + auto _gst_384 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1708 + t_lo_1709; diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu index ecc38ee65b5..7082e24aea8 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2246,15 +2269,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2275,15 +2304,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2304,15 +2339,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2333,15 +2374,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2502,15 +2573,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2531,15 +2608,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2560,15 +2643,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2589,15 +2678,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2636,15 +2731,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2665,15 +2766,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2694,15 +2801,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2723,15 +2836,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2802,37 +2921,37 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2965,19 +3084,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; - float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0) + dgf_0; + float qe = q_t0_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr)) + dgf_0; dgo[0] = dg_o_0; int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_248 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_249 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -3003,19 +3127,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_1 = dk2_1 * bt_t_1; float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; - float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1) + dgf_1; + float qe_0 = q_t0_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0)) + dgf_1; dgo[1] = dg_o_1; int qoff_1 = t_1 * qrs; if (t_1 < clen) { { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_250 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_251 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3041,19 +3170,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; - float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2) + dgf_2; + float qe_1 = q_t0_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1)) + dgf_2; dgo[2] = dg_o_2; int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_252 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_253 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3079,19 +3213,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; - float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3) + dgf_3; + float qe_2 = q_t0_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2)) + dgf_3; dgo[3] = dg_o_3; int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_254 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_255 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3117,19 +3256,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; - float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4) + dgf_4; + float qe_3 = q_t0_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3)) + dgf_4; dgo[4] = dg_o_4; int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_256 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_257 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3155,19 +3299,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; - float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5) + dgf_5; + float qe_4 = q_t0_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4)) + dgf_5; dgo[5] = dg_o_5; int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_258 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_259 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3193,19 +3342,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; - float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6) + dgf_6; + float qe_5 = q_t0_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5)) + dgf_6; dgo[6] = dg_o_6; int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_260 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_261 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3231,19 +3385,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; - float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7) + dgf_7; + float qe_6 = q_t0_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6)) + dgf_7; dgo[7] = dg_o_7; int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_262 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_263 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_7; } } float s_run[1]; @@ -3272,31 +3431,31 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3355,19 +3514,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; - float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8) + dgf_8; + float qe_7 = q_t0_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7)) + dgf_8; dgo_0[0] = dg_o_8; int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_273 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_274 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3393,19 +3557,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; - float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9) + dgf_9; + float qe_8 = q_t0_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8)) + dgf_9; dgo_0[1] = dg_o_9; int qoff_9 = t_9 * qrs; if (t_9 < clen) { { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_275 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_276 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3431,19 +3600,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; - float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10) + dgf_10; + float qe_9 = q_t0_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9)) + dgf_10; dgo_0[2] = dg_o_10; int qoff_10 = t_10 * qrs; if (t_10 < clen) { { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_277 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_278 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3469,19 +3643,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; - float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11) + dgf_11; + float qe_10 = q_t0_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10)) + dgf_11; dgo_0[3] = dg_o_11; int qoff_11 = t_11 * qrs; if (t_11 < clen) { { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_279 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_280 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3507,19 +3686,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; - float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12) + dgf_12; + float qe_11 = q_t0_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11)) + dgf_12; dgo_0[4] = dg_o_12; int qoff_12 = t_12 * qrs; if (t_12 < clen) { { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_281 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_282 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3545,19 +3729,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; - float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13) + dgf_13; + float qe_12 = q_t0_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12)) + dgf_13; dgo_0[5] = dg_o_13; int qoff_13 = t_13 * qrs; if (t_13 < clen) { { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_283 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_284 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3583,19 +3772,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; - float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14) + dgf_14; + float qe_13 = q_t0_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13)) + dgf_14; dgo_0[6] = dg_o_14; int qoff_14 = t_14 * qrs; if (t_14 < clen) { { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_285 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_286 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3621,19 +3815,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; - float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15) + dgf_15; + float qe_14 = q_t0_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14)) + dgf_15; dgo_0[7] = dg_o_15; int qoff_15 = t_15 * qrs; if (t_15 < clen) { { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_287 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_288 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_15; } } float s_run_0[1]; @@ -3662,36 +3861,36 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_289; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_290; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_291; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_292; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_293; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_294; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -3709,9 +3908,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_0 = offset + sfx[0]; if (t_16 < clen) { { - auto _gst_267 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_0; + auto _gst_299 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_0; } } int t_17 = 48 + 8 * th + 1; @@ -3719,9 +3918,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_1 = offset + sfx[1]; if (t_17 < clen) { { - auto _gst_268 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1; + auto _gst_300 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1; } } int t_18 = 48 + 8 * th + 2; @@ -3729,9 +3928,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_2 = offset + sfx[2]; if (t_18 < clen) { { - auto _gst_269 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_2; + auto _gst_301 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_2; } } int t_19 = 48 + 8 * th + 3; @@ -3739,9 +3938,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_3 = offset + sfx[3]; if (t_19 < clen) { { - auto _gst_270 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_3; + auto _gst_302 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_3; } } int t_20 = 48 + 8 * th + 4; @@ -3749,9 +3948,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_4 = offset + sfx[4]; if (t_20 < clen) { { - auto _gst_271 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_4; + auto _gst_303 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_4; } } int t_21 = 48 + 8 * th + 5; @@ -3759,9 +3958,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_5 = offset + sfx[5]; if (t_21 < clen) { { - auto _gst_272 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_5; + auto _gst_304 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_5; } } int t_22 = 48 + 8 * th + 6; @@ -3769,9 +3968,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_6 = offset + sfx[6]; if (t_22 < clen) { { - auto _gst_273 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_6; + auto _gst_305 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_6; } } int t_23 = 48 + 8 * th + 7; @@ -3779,9 +3978,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_7 = offset + sfx[7]; if (t_23 < clen) { { - auto _gst_274 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_7; + auto _gst_306 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3799,9 +3998,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_8 = offset_0 + sfx_0[0]; if (t_24 < clen) { { - auto _gst_275 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_8; + auto _gst_307 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_8; } } int t_25 = 32 + 8 * th + 1; @@ -3809,9 +4008,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_9 = offset_0 + sfx_0[1]; if (t_25 < clen) { { - auto _gst_276 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_9; + auto _gst_308 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_9; } } int t_26 = 32 + 8 * th + 2; @@ -3819,9 +4018,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_10 = offset_0 + sfx_0[2]; if (t_26 < clen) { { - auto _gst_277 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_10; + auto _gst_309 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_10; } } int t_27 = 32 + 8 * th + 3; @@ -3829,9 +4028,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_11 = offset_0 + sfx_0[3]; if (t_27 < clen) { { - auto _gst_278 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_11; + auto _gst_310 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_11; } } int t_28 = 32 + 8 * th + 4; @@ -3839,9 +4038,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_12 = offset_0 + sfx_0[4]; if (t_28 < clen) { { - auto _gst_279 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_12; + auto _gst_311 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_12; } } int t_29 = 32 + 8 * th + 5; @@ -3849,9 +4048,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_13 = offset_0 + sfx_0[5]; if (t_29 < clen) { { - auto _gst_280 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_13; + auto _gst_312 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_13; } } int t_30 = 32 + 8 * th + 6; @@ -3859,9 +4058,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_14 = offset_0 + sfx_0[6]; if (t_30 < clen) { { - auto _gst_281 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_14; + auto _gst_313 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_14; } } int t_31 = 32 + 8 * th + 7; @@ -3869,15 +4068,15 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_15 = offset_0 + sfx_0[7]; if (t_31 < clen) { { - auto _gst_282 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_15; + auto _gst_314 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3936,19 +4135,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; - float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16) + dgf_16; + float qe_15 = q_t0_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15)) + dgf_16; dgo_1[0] = dg_o_16; int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_284 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_316 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_285 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_317 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_33 = 16 + 8 * th + 1; @@ -3974,19 +4178,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; - float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17) + dgf_17; + float qe_16 = q_t0_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16)) + dgf_17; dgo_1[1] = dg_o_17; int qoff_17 = t_33 * qrs; if (t_33 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_34 = 16 + 8 * th + 2; @@ -4012,19 +4221,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; - float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18) + dgf_18; + float qe_17 = q_t0_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17)) + dgf_18; dgo_1[2] = dg_o_18; int qoff_18 = t_34 * qrs; if (t_34 < clen) { { - auto _gst_288 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_320 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_289 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_321 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_35 = 16 + 8 * th + 3; @@ -4050,19 +4264,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; - float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19) + dgf_19; + float qe_18 = q_t0_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18)) + dgf_19; dgo_1[3] = dg_o_19; int qoff_19 = t_35 * qrs; if (t_35 < clen) { { - auto _gst_290 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_322 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_291 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_323 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_36 = 16 + 8 * th + 4; @@ -4088,19 +4307,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; - float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20) + dgf_20; + float qe_19 = q_t0_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19)) + dgf_20; dgo_1[4] = dg_o_20; int qoff_20 = t_36 * qrs; if (t_36 < clen) { { - auto _gst_292 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_324 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_293 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_325 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_37 = 16 + 8 * th + 5; @@ -4126,19 +4350,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; - float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21) + dgf_21; + float qe_20 = q_t0_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20)) + dgf_21; dgo_1[5] = dg_o_21; int qoff_21 = t_37 * qrs; if (t_37 < clen) { { - auto _gst_294 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_326 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_295 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_327 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_38 = 16 + 8 * th + 6; @@ -4164,19 +4393,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; - float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22) + dgf_22; + float qe_21 = q_t0_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21)) + dgf_22; dgo_1[6] = dg_o_22; int qoff_22 = t_38 * qrs; if (t_38 < clen) { { - auto _gst_296 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_328 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_297 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_329 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_39 = 16 + 8 * th + 7; @@ -4202,19 +4436,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; - float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23) + dgf_23; + float qe_22 = q_t0_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22)) + dgf_23; dgo_1[7] = dg_o_23; int qoff_23 = t_39 * qrs; if (t_39 < clen) { { - auto _gst_298 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_330 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_299 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_331 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_23; } } float s_run_1[1]; @@ -4243,37 +4482,37 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_332; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_333; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_334; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_335; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_336; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_337; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4291,9 +4530,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_16 = offset_1 + sfx_1[0]; if (t_40 < clen) { { - auto _gst_310 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_16; + auto _gst_342 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_16; } } int t_41 = 16 + 8 * th + 1; @@ -4301,9 +4540,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_17 = offset_1 + sfx_1[1]; if (t_41 < clen) { { - auto _gst_311 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_17; + auto _gst_343 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_17; } } int t_42 = 16 + 8 * th + 2; @@ -4311,9 +4550,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_18 = offset_1 + sfx_1[2]; if (t_42 < clen) { { - auto _gst_312 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_18; + auto _gst_344 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_18; } } int t_43 = 16 + 8 * th + 3; @@ -4321,9 +4560,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_19 = offset_1 + sfx_1[3]; if (t_43 < clen) { { - auto _gst_313 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_19; + auto _gst_345 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_19; } } int t_44 = 16 + 8 * th + 4; @@ -4331,9 +4570,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_20 = offset_1 + sfx_1[4]; if (t_44 < clen) { { - auto _gst_314 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_20; + auto _gst_346 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_20; } } int t_45 = 16 + 8 * th + 5; @@ -4341,9 +4580,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_21 = offset_1 + sfx_1[5]; if (t_45 < clen) { { - auto _gst_315 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_21; + auto _gst_347 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_21; } } int t_46 = 16 + 8 * th + 6; @@ -4351,9 +4590,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_22 = offset_1 + sfx_1[6]; if (t_46 < clen) { { - auto _gst_316 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_22; + auto _gst_348 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_22; } } int t_47 = 16 + 8 * th + 7; @@ -4361,15 +4600,15 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_23 = offset_1 + sfx_1[7]; if (t_47 < clen) { { - auto _gst_317 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_23; + auto _gst_349 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4422,19 +4661,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; - float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24) + dgf_24; + float qe_23 = q_t0_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23)) + dgf_24; dgo_2[0] = dg_o_24; int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_319 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_351 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_320 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_352 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_49 = 8 * th + 1; @@ -4460,19 +4704,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_25 = dk2_25 * bt_t_25; float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; - float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25) + dgf_25; + float qe_24 = q_t0_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24)) + dgf_25; dgo_2[1] = dg_o_25; int qoff_25 = t_49 * qrs; if (t_49 < clen) { { - auto _gst_321 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_353 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_322 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_354 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_50 = 8 * th + 2; @@ -4498,19 +4747,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_26 = dk2_26 * bt_t_26; float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; - float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26) + dgf_26; + float qe_25 = q_t0_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25)) + dgf_26; dgo_2[2] = dg_o_26; int qoff_26 = t_50 * qrs; if (t_50 < clen) { { - auto _gst_323 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_355 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_324 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_356 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_51 = 8 * th + 3; @@ -4536,19 +4790,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_27 = dk2_27 * bt_t_27; float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; - float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27) + dgf_27; + float qe_26 = q_t0_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26)) + dgf_27; dgo_2[3] = dg_o_27; int qoff_27 = t_51 * qrs; if (t_51 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_52 = 8 * th + 4; @@ -4574,19 +4833,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_28 = dk2_28 * bt_t_28; float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; - float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28) + dgf_28; + float qe_27 = q_t0_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27)) + dgf_28; dgo_2[4] = dg_o_28; int qoff_28 = t_52 * qrs; if (t_52 < clen) { { - auto _gst_327 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_359 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_328 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_360 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_53 = 8 * th + 5; @@ -4612,19 +4876,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_29 = dk2_29 * bt_t_29; float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; - float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29) + dgf_29; + float qe_28 = q_t0_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28)) + dgf_29; dgo_2[5] = dg_o_29; int qoff_29 = t_53 * qrs; if (t_53 < clen) { { - auto _gst_329 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_361 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_330 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_362 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_54 = 8 * th + 6; @@ -4650,19 +4919,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_30 = dk2_30 * bt_t_30; float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; - float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30) + dgf_30; + float qe_29 = q_t0_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29)) + dgf_30; dgo_2[6] = dg_o_30; int qoff_30 = t_54 * qrs; if (t_54 < clen) { { - auto _gst_331 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_363 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_332 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_364 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_55 = 8 * th + 7; @@ -4688,19 +4962,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_31 = dk2_31 * bt_t_31; float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; - float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31) + dgf_31; + float qe_30 = q_t0_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30)) + dgf_31; dgo_2[7] = dg_o_31; int qoff_31 = t_55 * qrs; if (t_55 < clen) { { - auto _gst_333 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_365 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_334 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_366 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_31; } } float s_run_2[1]; @@ -4729,35 +5008,35 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_367; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_368; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_369; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_370; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_371; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_372; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_2) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4775,9 +5054,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_24 = offset_2 + sfx_2[0]; if (t_56 < clen) { { - auto _gst_345 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_24; + auto _gst_377 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_24; } } int t_57 = 8 * th + 1; @@ -4785,9 +5064,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_25 = offset_2 + sfx_2[1]; if (t_57 < clen) { { - auto _gst_346 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_25; + auto _gst_378 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_25; } } int t_58 = 8 * th + 2; @@ -4795,9 +5074,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_26 = offset_2 + sfx_2[2]; if (t_58 < clen) { { - auto _gst_347 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_26; + auto _gst_379 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_26; } } int t_59 = 8 * th + 3; @@ -4805,9 +5084,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_27 = offset_2 + sfx_2[3]; if (t_59 < clen) { { - auto _gst_348 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_27; + auto _gst_380 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_27; } } int t_60 = 8 * th + 4; @@ -4815,9 +5094,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_28 = offset_2 + sfx_2[4]; if (t_60 < clen) { { - auto _gst_349 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_28; + auto _gst_381 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_28; } } int t_61 = 8 * th + 5; @@ -4825,9 +5104,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_29 = offset_2 + sfx_2[5]; if (t_61 < clen) { { - auto _gst_350 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_29; + auto _gst_382 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_29; } } int t_62 = 8 * th + 6; @@ -4835,9 +5114,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_30 = offset_2 + sfx_2[6]; if (t_62 < clen) { { - auto _gst_351 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_30; + auto _gst_383 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_30; } } int t_63 = 8 * th + 7; @@ -4845,9 +5124,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_31 = offset_2 + sfx_2[7]; if (t_63 < clen) { { - auto _gst_352 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_31; + auto _gst_384 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_31; } } off_acc[0] = off_acc[0] + t_hi_2 + t_lo_2; diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu index a07b0c5e214..95e9120961d 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu @@ -130,6 +130,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -172,6 +179,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2235,15 +2258,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2264,15 +2293,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2293,15 +2328,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2322,15 +2363,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2366,12 +2413,18 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2391,12 +2444,18 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2416,12 +2475,18 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2441,12 +2506,18 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2491,15 +2562,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2520,15 +2597,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2549,15 +2632,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2578,15 +2667,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2625,15 +2720,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2654,15 +2755,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2683,15 +2790,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2712,15 +2825,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2791,37 +2910,37 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2833,29 +2952,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2863,29 +2982,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2954,24 +3073,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; float q_r_0 = q_t0_0; - float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0)) + dgf_0; + float qe = q_r_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr))) + dgf_0; int goff_0 = t_0 * rs_g; int qoff_0 = t_0 * qrs; { - auto _gst_216 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast(_gst_216) + (0)) = dg_o_0; + auto _gst_248 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast(_gst_248) + (0)) = dg_o_0; } if (t_0 < clen) { { - auto _gst_217 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_249 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_218 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_250 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -2998,24 +3122,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; float q_r_1 = q_t0_1; - float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1)) + dgf_1; + float qe_0 = q_r_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0))) + dgf_1; int goff_1 = t_1 * rs_g; int qoff_1 = t_1 * qrs; { - auto _gst_219 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast(_gst_219) + (0)) = dg_o_1; + auto _gst_251 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast(_gst_251) + (0)) = dg_o_1; } if (t_1 < clen) { { - auto _gst_220 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_252 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_221 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_253 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3042,24 +3171,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; float q_r_2 = q_t0_2; - float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2)) + dgf_2; + float qe_1 = q_r_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1))) + dgf_2; int goff_2 = t_2 * rs_g; int qoff_2 = t_2 * qrs; { - auto _gst_222 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast(_gst_222) + (0)) = dg_o_2; + auto _gst_254 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast(_gst_254) + (0)) = dg_o_2; } if (t_2 < clen) { { - auto _gst_223 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_255 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_224 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_256 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3086,24 +3220,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; float q_r_3 = q_t0_3; - float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3)) + dgf_3; + float qe_2 = q_r_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2))) + dgf_3; int goff_3 = t_3 * rs_g; int qoff_3 = t_3 * qrs; { - auto _gst_225 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast(_gst_225) + (0)) = dg_o_3; + auto _gst_257 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast(_gst_257) + (0)) = dg_o_3; } if (t_3 < clen) { { - auto _gst_226 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_258 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_227 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_259 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3130,24 +3269,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; float q_r_4 = q_t0_4; - float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4)) + dgf_4; + float qe_3 = q_r_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3))) + dgf_4; int goff_4 = t_4 * rs_g; int qoff_4 = t_4 * qrs; { - auto _gst_228 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast(_gst_228) + (0)) = dg_o_4; + auto _gst_260 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast(_gst_260) + (0)) = dg_o_4; } if (t_4 < clen) { { - auto _gst_229 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_261 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_230 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_262 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3174,24 +3318,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; float q_r_5 = q_t0_5; - float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5)) + dgf_5; + float qe_4 = q_r_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4))) + dgf_5; int goff_5 = t_5 * rs_g; int qoff_5 = t_5 * qrs; { - auto _gst_231 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast(_gst_231) + (0)) = dg_o_5; + auto _gst_263 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast(_gst_263) + (0)) = dg_o_5; } if (t_5 < clen) { { - auto _gst_232 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_232)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_232) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_264 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_264)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_264) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_233 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_233)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_233) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_265 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_265)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3218,24 +3367,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; float q_r_6 = q_t0_6; - float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6)) + dgf_6; + float qe_5 = q_r_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5))) + dgf_6; int goff_6 = t_6 * rs_g; int qoff_6 = t_6 * qrs; { - auto _gst_234 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_234)); - *(reinterpret_cast(_gst_234) + (0)) = dg_o_6; + auto _gst_266 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_266)); + *(reinterpret_cast(_gst_266) + (0)) = dg_o_6; } if (t_6 < clen) { { - auto _gst_235 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_235)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_235) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_267 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_267)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_267) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_236 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_236)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_236) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_268 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_268)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3262,24 +3416,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; float q_r_7 = q_t0_7; - float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7)) + dgf_7; + float qe_6 = q_r_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6))) + dgf_7; int goff_7 = t_7 * rs_g; int qoff_7 = t_7 * qrs; { - auto _gst_237 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_237)); - *(reinterpret_cast(_gst_237) + (0)) = dg_o_7; + auto _gst_269 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_269)); + *(reinterpret_cast(_gst_269) + (0)) = dg_o_7; } if (t_7 < clen) { { - auto _gst_238 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_238)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_238) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_270 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_270)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_270) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_239 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_239)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_239) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_271 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_271)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dk_o_7; } } int b4 = lane_1 >> 4 & 1; @@ -3287,31 +3446,31 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_240; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_272; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_241 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_241; + float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_273; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_242 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_242; + float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_274; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_243 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_243; + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_275; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_244 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_244; + float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_276; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_245 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_245; + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_277; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_246 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_246; - float _shfl_xor_sync_247 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_247; - float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_248; + float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_278; + float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_279; + float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_280; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3370,24 +3529,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; float q_r_8 = q_t0_8; - float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8)) + dgf_8; + float qe_7 = q_r_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7))) + dgf_8; int goff_8 = t_8 * rs_g; int qoff_8 = t_8 * qrs; { - auto _gst_249 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast(_gst_249) + (0)) = dg_o_8; + auto _gst_281 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast(_gst_281) + (0)) = dg_o_8; } if (t_8 < clen) { { - auto _gst_250 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_282 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_251 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_283 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3414,24 +3578,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; float q_r_9 = q_t0_9; - float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9)) + dgf_9; + float qe_8 = q_r_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8))) + dgf_9; int goff_9 = t_9 * rs_g; int qoff_9 = t_9 * qrs; { - auto _gst_252 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast(_gst_252) + (0)) = dg_o_9; + auto _gst_284 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast(_gst_284) + (0)) = dg_o_9; } if (t_9 < clen) { { - auto _gst_253 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_285 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_254 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_286 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3458,24 +3627,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; float q_r_10 = q_t0_10; - float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10)) + dgf_10; + float qe_9 = q_r_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9))) + dgf_10; int goff_10 = t_10 * rs_g; int qoff_10 = t_10 * qrs; { - auto _gst_255 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast(_gst_255) + (0)) = dg_o_10; + auto _gst_287 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast(_gst_287) + (0)) = dg_o_10; } if (t_10 < clen) { { - auto _gst_256 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_288 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_257 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_257)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_289 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_289)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3502,24 +3676,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; float q_r_11 = q_t0_11; - float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11)) + dgf_11; + float qe_10 = q_r_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10))) + dgf_11; int goff_11 = t_11 * rs_g; int qoff_11 = t_11 * qrs; { - auto _gst_258 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_258)); - *(reinterpret_cast(_gst_258) + (0)) = dg_o_11; + auto _gst_290 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_290)); + *(reinterpret_cast(_gst_290) + (0)) = dg_o_11; } if (t_11 < clen) { { - auto _gst_259 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_259)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_291 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_291)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_260 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_260)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_292 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_292)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3546,24 +3725,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; float q_r_12 = q_t0_12; - float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12)) + dgf_12; + float qe_11 = q_r_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11))) + dgf_12; int goff_12 = t_12 * rs_g; int qoff_12 = t_12 * qrs; { - auto _gst_261 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_261)); - *(reinterpret_cast(_gst_261) + (0)) = dg_o_12; + auto _gst_293 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_293)); + *(reinterpret_cast(_gst_293) + (0)) = dg_o_12; } if (t_12 < clen) { { - auto _gst_262 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_262)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_294 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_294)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_263 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_263)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_295 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_295)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3590,24 +3774,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; float q_r_13 = q_t0_13; - float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13)) + dgf_13; + float qe_12 = q_r_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12))) + dgf_13; int goff_13 = t_13 * rs_g; int qoff_13 = t_13 * qrs; { - auto _gst_264 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_264)); - *(reinterpret_cast(_gst_264) + (0)) = dg_o_13; + auto _gst_296 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_296)); + *(reinterpret_cast(_gst_296) + (0)) = dg_o_13; } if (t_13 < clen) { { - auto _gst_265 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_265)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_297 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_297)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_266 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_266)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_266) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_298 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_298)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3634,24 +3823,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; float q_r_14 = q_t0_14; - float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14)) + dgf_14; + float qe_13 = q_r_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13))) + dgf_14; int goff_14 = t_14 * rs_g; int qoff_14 = t_14 * qrs; { - auto _gst_267 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_o_14; + auto _gst_299 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_o_14; } if (t_14 < clen) { { - auto _gst_268 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_300 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_300) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_269 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_269) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_301 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3678,24 +3872,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; float q_r_15 = q_t0_15; - float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15)) + dgf_15; + float qe_14 = q_r_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14))) + dgf_15; int goff_15 = t_15 * rs_g; int qoff_15 = t_15 * qrs; { - auto _gst_270 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_o_15; + auto _gst_302 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_o_15; } if (t_15 < clen) { { - auto _gst_271 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_303 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_303) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_272 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_272) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_304 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dk_o_15; } } int b4_0 = lane_1 >> 4 & 1; @@ -3703,31 +3902,31 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_273; + float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_305; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_274; + float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_306; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_275; + float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_307; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_276; + float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_308; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_277; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_309; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_278; + float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_310; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_279; - float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_280; - float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_281; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_311; + float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_312; + float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_313; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); mbarrier_wait(p1b_ready_addr, 0); @@ -3787,24 +3986,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; float q_r_16 = q_t0_16; - float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16)) + dgf_16; + float qe_15 = q_r_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15))) + dgf_16; int goff_16 = t_16 * rs_g; int qoff_16 = t_16 * qrs; { - auto _gst_282 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_o_16; + auto _gst_314 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_o_16; } if (t_16 < clen) { { - auto _gst_283 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_283)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_315 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_315)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_315) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_284 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_316 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_17 = 16 + 8 * th + 1; @@ -3831,24 +4035,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; float q_r_17 = q_t0_17; - float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17)) + dgf_17; + float qe_16 = q_r_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16))) + dgf_17; int goff_17 = t_17 * rs_g; int qoff_17 = t_17 * qrs; { - auto _gst_285 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast(_gst_285) + (0)) = dg_o_17; + auto _gst_317 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast(_gst_317) + (0)) = dg_o_17; } if (t_17 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_18 = 16 + 8 * th + 2; @@ -3875,24 +4084,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; float q_r_18 = q_t0_18; - float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18)) + dgf_18; + float qe_17 = q_r_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17))) + dgf_18; int goff_18 = t_18 * rs_g; int qoff_18 = t_18 * qrs; { - auto _gst_288 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast(_gst_288) + (0)) = dg_o_18; + auto _gst_320 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = dg_o_18; } if (t_18 < clen) { { - auto _gst_289 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_321 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_290 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_322 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_19 = 16 + 8 * th + 3; @@ -3919,24 +4133,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; float q_r_19 = q_t0_19; - float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19)) + dgf_19; + float qe_18 = q_r_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18))) + dgf_19; int goff_19 = t_19 * rs_g; int qoff_19 = t_19 * qrs; { - auto _gst_291 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast(_gst_291) + (0)) = dg_o_19; + auto _gst_323 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast(_gst_323) + (0)) = dg_o_19; } if (t_19 < clen) { { - auto _gst_292 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_324 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_293 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_325 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_20 = 16 + 8 * th + 4; @@ -3963,24 +4182,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; float q_r_20 = q_t0_20; - float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20)) + dgf_20; + float qe_19 = q_r_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19))) + dgf_20; int goff_20 = t_20 * rs_g; int qoff_20 = t_20 * qrs; { - auto _gst_294 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast(_gst_294) + (0)) = dg_o_20; + auto _gst_326 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast(_gst_326) + (0)) = dg_o_20; } if (t_20 < clen) { { - auto _gst_295 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_327 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_296 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_328 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_21 = 16 + 8 * th + 5; @@ -4007,24 +4231,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; float q_r_21 = q_t0_21; - float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21)) + dgf_21; + float qe_20 = q_r_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20))) + dgf_21; int goff_21 = t_21 * rs_g; int qoff_21 = t_21 * qrs; { - auto _gst_297 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast(_gst_297) + (0)) = dg_o_21; + auto _gst_329 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast(_gst_329) + (0)) = dg_o_21; } if (t_21 < clen) { { - auto _gst_298 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_330 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_299 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_331 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_22 = 16 + 8 * th + 6; @@ -4051,24 +4280,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; float q_r_22 = q_t0_22; - float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22)) + dgf_22; + float qe_21 = q_r_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21))) + dgf_22; int goff_22 = t_22 * rs_g; int qoff_22 = t_22 * qrs; { - auto _gst_300 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_300)); - *(reinterpret_cast(_gst_300) + (0)) = dg_o_22; + auto _gst_332 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_332)); + *(reinterpret_cast(_gst_332) + (0)) = dg_o_22; } if (t_22 < clen) { { - auto _gst_301 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_301)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_333 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_333)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_302 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_302)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_302) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_334 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_334)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_23 = 16 + 8 * th + 7; @@ -4095,24 +4329,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; float q_r_23 = q_t0_23; - float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23)) + dgf_23; + float qe_22 = q_r_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22))) + dgf_23; int goff_23 = t_23 * rs_g; int qoff_23 = t_23 * qrs; { - auto _gst_303 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_303)); - *(reinterpret_cast(_gst_303) + (0)) = dg_o_23; + auto _gst_335 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_335)); + *(reinterpret_cast(_gst_335) + (0)) = dg_o_23; } if (t_23 < clen) { { - auto _gst_304 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_304)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_336 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_336)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_336) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_305 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_305)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_305) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_337 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_337)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dk_o_23; } } int b4_1 = lane_1 >> 4 & 1; @@ -4120,31 +4359,31 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_306; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_338; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_307; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_339; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_308; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_340; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_309; + float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_341; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_310; + float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_342; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_311; + float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_343; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_312; - float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_313; - float _shfl_xor_sync_314 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_314; + float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_344; + float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_345; + float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_346; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); mbarrier_wait(epi_full_addr + (th) * 8, 0); @@ -4198,24 +4437,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; float q_r_24 = q_t0_24; - float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24)) + dgf_24; + float qe_23 = q_r_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23))) + dgf_24; int goff_24 = t_24 * rs_g; int qoff_24 = t_24 * qrs; { - auto _gst_315 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_o_24; + auto _gst_347 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_o_24; } if (t_24 < clen) { { - auto _gst_316 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_348 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_348) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_317 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_349 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_349) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_25 = 8 * th + 1; @@ -4242,24 +4486,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; float q_r_25 = q_t0_25; - float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25)) + dgf_25; + float qe_24 = q_r_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24))) + dgf_25; int goff_25 = t_25 * rs_g; int qoff_25 = t_25 * qrs; { - auto _gst_318 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = dg_o_25; + auto _gst_350 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_350)); + *(reinterpret_cast(_gst_350) + (0)) = dg_o_25; } if (t_25 < clen) { { - auto _gst_319 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_351 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_320 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_352 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_26 = 8 * th + 2; @@ -4286,24 +4535,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; float q_r_26 = q_t0_26; - float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26)) + dgf_26; + float qe_25 = q_r_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25))) + dgf_26; int goff_26 = t_26 * rs_g; int qoff_26 = t_26 * qrs; { - auto _gst_321 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast(_gst_321) + (0)) = dg_o_26; + auto _gst_353 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast(_gst_353) + (0)) = dg_o_26; } if (t_26 < clen) { { - auto _gst_322 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_354 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_323 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_355 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_27 = 8 * th + 3; @@ -4330,24 +4584,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; float q_r_27 = q_t0_27; - float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27)) + dgf_27; + float qe_26 = q_r_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26))) + dgf_27; int goff_27 = t_27 * rs_g; int qoff_27 = t_27 * qrs; { - auto _gst_324 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast(_gst_324) + (0)) = dg_o_27; + auto _gst_356 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast(_gst_356) + (0)) = dg_o_27; } if (t_27 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_28 = 8 * th + 4; @@ -4374,24 +4633,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; float q_r_28 = q_t0_28; - float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28)) + dgf_28; + float qe_27 = q_r_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27))) + dgf_28; int goff_28 = t_28 * rs_g; int qoff_28 = t_28 * qrs; { - auto _gst_327 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast(_gst_327) + (0)) = dg_o_28; + auto _gst_359 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast(_gst_359) + (0)) = dg_o_28; } if (t_28 < clen) { { - auto _gst_328 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_360 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_329 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_361 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_29 = 8 * th + 5; @@ -4418,24 +4682,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; float q_r_29 = q_t0_29; - float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29)) + dgf_29; + float qe_28 = q_r_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28))) + dgf_29; int goff_29 = t_29 * rs_g; int qoff_29 = t_29 * qrs; { - auto _gst_330 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast(_gst_330) + (0)) = dg_o_29; + auto _gst_362 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast(_gst_362) + (0)) = dg_o_29; } if (t_29 < clen) { { - auto _gst_331 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_363 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_332 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_364 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_30 = 8 * th + 6; @@ -4462,24 +4731,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; float q_r_30 = q_t0_30; - float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30)) + dgf_30; + float qe_29 = q_r_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29))) + dgf_30; int goff_30 = t_30 * rs_g; int qoff_30 = t_30 * qrs; { - auto _gst_333 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast(_gst_333) + (0)) = dg_o_30; + auto _gst_365 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast(_gst_365) + (0)) = dg_o_30; } if (t_30 < clen) { { - auto _gst_334 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_366 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_335 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_335)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_335) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_367 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_367)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_367) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_31 = 8 * th + 7; @@ -4506,24 +4780,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; float q_r_31 = q_t0_31; - float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31)) + dgf_31; + float qe_30 = q_r_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30))) + dgf_31; int goff_31 = t_31 * rs_g; int qoff_31 = t_31 * qrs; { - auto _gst_336 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_336)); - *(reinterpret_cast(_gst_336) + (0)) = dg_o_31; + auto _gst_368 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_368)); + *(reinterpret_cast(_gst_368) + (0)) = dg_o_31; } if (t_31 < clen) { { - auto _gst_337 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_337)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_369 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_369)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_369) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_338 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_338)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_338) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_370 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_370)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_370) + (0)) = (__nv_bfloat16)dk_o_31; } } int b4_2 = lane_1 >> 4 & 1; @@ -4531,31 +4810,31 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_371; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_372; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_341; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_373; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_342; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_374; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_343; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_375; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_344; + float _shfl_xor_sync_376 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_376; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_345; - float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_346; - float _shfl_xor_sync_347 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_347; + float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_377; + float _shfl_xor_sync_378 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_378; + float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_379; if ((lane_1 & 3) == 0) { int e_star = lane_1 >> 2 & 7; pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_2; diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu index 637fa7a75e0..9fa69d65b88 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -199,6 +200,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2267,12 +2284,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_0); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_0 : __fmaf_rn(dl, _rcp_170, e2_0)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2296,12 +2319,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_1); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_1, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_1, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_1 : __fmaf_rn(dl_1, _rcp_174, e2_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2325,12 +2354,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_2); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_2, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_2, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_2 : __fmaf_rn(dl_3, _rcp_178, e2_2)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2354,12 +2389,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_3); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_3, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_3, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_3 : __fmaf_rn(dl_5, _rcp_182, e2_3)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2395,12 +2436,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2420,12 +2467,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2445,12 +2498,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2470,12 +2529,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2523,12 +2588,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2552,12 +2623,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_1 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2581,12 +2658,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_2 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2610,12 +2693,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_3 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2657,12 +2746,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_4 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2686,12 +2781,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_5 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2715,12 +2816,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_6 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2744,12 +2851,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_7 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2820,37 +2933,37 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 d2_56 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_56.x); float e1_12 = approx_exp2(d2_56.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_57.x); float e1_13 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_58.x); float e1_14 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_59.x); float e1_15 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2862,29 +2975,29 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2892,29 +3005,29 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2933,409 +3046,145 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float r_247[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_247[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 4096 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 4096 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); float gn = r_247[0]; - float gc_888 = gn - 21.640425f; + float gc_0 = gn - 21.640425f; int eb = 98304 + th * -32768; int ew = eb >> 2; - int qbase_889 = pool_f32_addr + (unsigned int)(eb + 12288 + dh_lane * 1024); + int qbase_0 = pool_f32_addr + (unsigned int)(eb + 12288 + dh_lane * 1024); int sb8 = 32 + 8 * th; int gb8 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8 * 128); int kb8 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8 * 128); - float dbp[8]; - float _tmem_load_7[8]; - tmem_ld_x8(&_tmem_load_7[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float _tmem_load_8[8]; - tmem_ld_x8(&_tmem_load_8[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float _tmem_load_9[8]; - tmem_ld_x8(&_tmem_load_9[0], taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)); - float _tmem_load_10[8]; - tmem_ld_x8(&_tmem_load_10[0], taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th)); - float _tmem_load_11[8]; - tmem_ld_x8(&_tmem_load_11[0], taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th)); - float btv[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[0])), "=r"(*reinterpret_cast(&btv[(0) + 1])), "=r"(*reinterpret_cast(&btv[(0) + 2])), "=r"(*reinterpret_cast(&btv[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[4])), "=r"(*reinterpret_cast(&btv[(4) + 1])), "=r"(*reinterpret_cast(&btv[(4) + 2])), "=r"(*reinterpret_cast(&btv[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int c_qa = 128 + 8 * th; + int c_ka = 144 + 8 * th; + int c_qd = 192 + 8 * th; + int c_kd = 208 + 8 * th; + int c_dk = 32 + 8 * th; + int c_bp = c_ka; float s_run[1]; s_run[0] = 0.0f; - int t_1_0 = 32 + 8 * th + 7; - float r_248[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_0 = r_248[0]; - unsigned int r_249[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_0 = __uint_as_float(r_249[0] << 16); - unsigned int r_250[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_0 = __uint_as_float(r_250[0] << 16); - float q_t_2_0 = ((t_1_0 >= clen) ? 0.0f : q_t_1_0); - float dqf_1_0 = pool_f32[ew + 896 + d_lane]; - float dkf_1_0 = pool_f32[ew + 1920 + d_lane]; - float dgf_1_0 = pool_f32[ew + 2944 + d_lane]; - float bt_t_1_0 = btv[7]; - float e_off_1_0 = approx_exp2(g_t_1_0 - gn); - float e_dg_1_0 = approx_exp2(g_t_1_0 - gc_888); - float dq2_1_0 = __fmaf_rn(_tmem_load_7[7], e_off_1_0, _tmem_load_9[7] * e_dg_1_0); - float dk2_1_0 = __fmaf_rn(_tmem_load_8[7], e_off_1_0, _tmem_load_10[7] * e_dg_1_0); - dbp[7] = dk2_1_0 * k_t_1_0; - float dk2b_1_0 = dk2_1_0 * bt_t_1_0; - float dq_o_1_0 = dq2_1_0 + dqf_1_0; - float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; - s_run[0] = s_run[0] + dg_o_1_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 7)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_0 = t_1_0 * qrs; - if (t_1_0 < clen) { - { - auto _gst_216 = dq_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_1_0; - } - { - auto _gst_217 = dk_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_1_0; - } - } - int t_1_1 = 32 + 8 * th + 6; - float r_251[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_1 = r_251[0]; - unsigned int r_252[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((kb8 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_1 = __uint_as_float(r_252[0] << 16); - unsigned int r_253[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_1 = __uint_as_float(r_253[0] << 16); - float q_t_2_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_1); - float dqf_1_1 = pool_f32[ew + 768 + d_lane]; - float dkf_1_1 = pool_f32[ew + 1792 + d_lane]; - float dgf_1_1 = pool_f32[ew + 2816 + d_lane]; - float bt_t_1_1 = btv[6]; - float e_off_1_1 = approx_exp2(g_t_1_1 - gn); - float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_888); - float dq2_1_1 = __fmaf_rn(_tmem_load_7[6], e_off_1_1, _tmem_load_9[6] * e_dg_1_1); - float dk2_1_1 = __fmaf_rn(_tmem_load_8[6], e_off_1_1, _tmem_load_10[6] * e_dg_1_1); - dbp[6] = dk2_1_1 * k_t_1_1; - float dk2b_1_1 = dk2_1_1 * bt_t_1_1; - float dq_o_1_1 = dq2_1_1 + dqf_1_1; - float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; - s_run[0] = s_run[0] + dg_o_1_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 6)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_1 = t_1_1 * qrs; - if (t_1_1 < clen) { - { - auto _gst_218 = dq_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1_1; - } - { - auto _gst_219 = dk_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1_1; - } - } - int t_1_2 = 32 + 8 * th + 5; - float r_254[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_2 = r_254[0]; - unsigned int r_255[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((kb8 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_2 = __uint_as_float(r_255[0] << 16); - unsigned int r_256[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_2 = __uint_as_float(r_256[0] << 16); - float q_t_2_2 = ((t_1_2 >= clen) ? 0.0f : q_t_1_2); - float dqf_1_2 = pool_f32[ew + 640 + d_lane]; - float dkf_1_2 = pool_f32[ew + 1664 + d_lane]; - float dgf_1_2 = pool_f32[ew + 2688 + d_lane]; - float bt_t_1_2 = btv[5]; - float e_off_1_2 = approx_exp2(g_t_1_2 - gn); - float e_dg_1_2 = approx_exp2(g_t_1_2 - gc_888); - float dq2_1_2 = __fmaf_rn(_tmem_load_7[5], e_off_1_2, _tmem_load_9[5] * e_dg_1_2); - float dk2_1_2 = __fmaf_rn(_tmem_load_8[5], e_off_1_2, _tmem_load_10[5] * e_dg_1_2); - dbp[5] = dk2_1_2 * k_t_1_2; - float dk2b_1_2 = dk2_1_2 * bt_t_1_2; - float dq_o_1_2 = dq2_1_2 + dqf_1_2; - float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; - s_run[0] = s_run[0] + dg_o_1_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 5)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_2 = t_1_2 * qrs; - if (t_1_2 < clen) { - { - auto _gst_220 = dq_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1_2; - } - { - auto _gst_221 = dk_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1_2; - } - } - int t_1_3 = 32 + 8 * th + 4; - float r_257[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_3 = r_257[0]; - unsigned int r_258[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((kb8 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_3 = __uint_as_float(r_258[0] << 16); - unsigned int r_259[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_3 = __uint_as_float(r_259[0] << 16); - float q_t_2_3 = ((t_1_3 >= clen) ? 0.0f : q_t_1_3); - float dqf_1_3 = pool_f32[ew + 512 + d_lane]; - float dkf_1_3 = pool_f32[ew + 1536 + d_lane]; - float dgf_1_3 = pool_f32[ew + 2560 + d_lane]; - float bt_t_1_3 = btv[4]; - float e_off_1_3 = approx_exp2(g_t_1_3 - gn); - float e_dg_1_3 = approx_exp2(g_t_1_3 - gc_888); - float dq2_1_3 = __fmaf_rn(_tmem_load_7[4], e_off_1_3, _tmem_load_9[4] * e_dg_1_3); - float dk2_1_3 = __fmaf_rn(_tmem_load_8[4], e_off_1_3, _tmem_load_10[4] * e_dg_1_3); - dbp[4] = dk2_1_3 * k_t_1_3; - float dk2b_1_3 = dk2_1_3 * bt_t_1_3; - float dq_o_1_3 = dq2_1_3 + dqf_1_3; - float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; - s_run[0] = s_run[0] + dg_o_1_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 4)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_3 = t_1_3 * qrs; - if (t_1_3 < clen) { - { - auto _gst_222 = dq_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_1_3; - } - { - auto _gst_223 = dk_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_1_3; - } - } - int t_1_4 = 32 + 8 * th + 3; - float r_260[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_4 = r_260[0]; - unsigned int r_261[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((kb8 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_4 = __uint_as_float(r_261[0] << 16); - unsigned int r_262[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_4 = __uint_as_float(r_262[0] << 16); - float q_t_2_4 = ((t_1_4 >= clen) ? 0.0f : q_t_1_4); - float dqf_1_4 = pool_f32[ew + 384 + d_lane]; - float dkf_1_4 = pool_f32[ew + 1408 + d_lane]; - float dgf_1_4 = pool_f32[ew + 2432 + d_lane]; - float bt_t_1_4 = btv[3]; - float e_off_1_4 = approx_exp2(g_t_1_4 - gn); - float e_dg_1_4 = approx_exp2(g_t_1_4 - gc_888); - float dq2_1_4 = __fmaf_rn(_tmem_load_7[3], e_off_1_4, _tmem_load_9[3] * e_dg_1_4); - float dk2_1_4 = __fmaf_rn(_tmem_load_8[3], e_off_1_4, _tmem_load_10[3] * e_dg_1_4); - dbp[3] = dk2_1_4 * k_t_1_4; - float dk2b_1_4 = dk2_1_4 * bt_t_1_4; - float dq_o_1_4 = dq2_1_4 + dqf_1_4; - float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; - s_run[0] = s_run[0] + dg_o_1_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 3)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_4 = t_1_4 * qrs; - if (t_1_4 < clen) { - { - auto _gst_224 = dq_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_1_4; - } - { - auto _gst_225 = dk_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_1_4; - } - } - int t_1_5 = 32 + 8 * th + 2; - float r_263[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_5 = r_263[0]; - unsigned int r_264[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((kb8 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_5 = __uint_as_float(r_264[0] << 16); - unsigned int r_265[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_5 = __uint_as_float(r_265[0] << 16); - float q_t_2_5 = ((t_1_5 >= clen) ? 0.0f : q_t_1_5); - float dqf_1_5 = pool_f32[ew + 256 + d_lane]; - float dkf_1_5 = pool_f32[ew + 1280 + d_lane]; - float dgf_1_5 = pool_f32[ew + 2304 + d_lane]; - float bt_t_1_5 = btv[2]; - float e_off_1_5 = approx_exp2(g_t_1_5 - gn); - float e_dg_1_5 = approx_exp2(g_t_1_5 - gc_888); - float dq2_1_5 = __fmaf_rn(_tmem_load_7[2], e_off_1_5, _tmem_load_9[2] * e_dg_1_5); - float dk2_1_5 = __fmaf_rn(_tmem_load_8[2], e_off_1_5, _tmem_load_10[2] * e_dg_1_5); - dbp[2] = dk2_1_5 * k_t_1_5; - float dk2b_1_5 = dk2_1_5 * bt_t_1_5; - float dq_o_1_5 = dq2_1_5 + dqf_1_5; - float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; - s_run[0] = s_run[0] + dg_o_1_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 2)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_5 = t_1_5 * qrs; - if (t_1_5 < clen) { - { - auto _gst_226 = dq_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_1_5; - } - { - auto _gst_227 = dk_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_1_5; - } - } - int t_1_6 = 32 + 8 * th + 1; - float r_266[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_6 = r_266[0]; - unsigned int r_267[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_6 = __uint_as_float(r_267[0] << 16); - unsigned int r_268[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_6 = __uint_as_float(r_268[0] << 16); - float q_t_2_6 = ((t_1_6 >= clen) ? 0.0f : q_t_1_6); - float dqf_1_6 = pool_f32[ew + 128 + d_lane]; - float dkf_1_6 = pool_f32[ew + 1152 + d_lane]; - float dgf_1_6 = pool_f32[ew + 2176 + d_lane]; - float bt_t_1_6 = btv[1]; - float e_off_1_6 = approx_exp2(g_t_1_6 - gn); - float e_dg_1_6 = approx_exp2(g_t_1_6 - gc_888); - float dq2_1_6 = __fmaf_rn(_tmem_load_7[1], e_off_1_6, _tmem_load_9[1] * e_dg_1_6); - float dk2_1_6 = __fmaf_rn(_tmem_load_8[1], e_off_1_6, _tmem_load_10[1] * e_dg_1_6); - dbp[1] = dk2_1_6 * k_t_1_6; - float dk2b_1_6 = dk2_1_6 * bt_t_1_6; - float dq_o_1_6 = dq2_1_6 + dqf_1_6; - float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; - s_run[0] = s_run[0] + dg_o_1_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 1)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_6 = t_1_6 * qrs; - if (t_1_6 < clen) { - { - auto _gst_228 = dq_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_1_6; - } - { - auto _gst_229 = dk_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_1_6; - } - } - int t_1_7 = 32 + 8 * th; - float r_269[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_7 = r_269[0]; - unsigned int r_270[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((kb8 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_7 = __uint_as_float(r_270[0] << 16); - unsigned int r_271[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_7 = __uint_as_float(r_271[0] << 16); - float q_t_2_7 = ((t_1_7 >= clen) ? 0.0f : q_t_1_7); - float dqf_1_7 = pool_f32[ew + d_lane]; - float dkf_1_7 = pool_f32[ew + 1024 + d_lane]; - float dgf_1_7 = pool_f32[ew + 2048 + d_lane]; - float bt_t_1_7 = btv[0]; - float e_off_1_7 = approx_exp2(g_t_1_7 - gn); - float e_dg_1_7 = approx_exp2(g_t_1_7 - gc_888); - float dq2_1_7 = __fmaf_rn(_tmem_load_7[0], e_off_1_7, _tmem_load_9[0] * e_dg_1_7); - float dk2_1_7 = __fmaf_rn(_tmem_load_8[0], e_off_1_7, _tmem_load_10[0] * e_dg_1_7); - dbp[0] = dk2_1_7 * k_t_1_7; - float dk2b_1_7 = dk2_1_7 * bt_t_1_7; - float dq_o_1_7 = dq2_1_7 + dqf_1_7; - float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; - s_run[0] = s_run[0] + dg_o_1_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_7 = t_1_7 * qrs; - if (t_1_7 < clen) { - { - auto _gst_230 = dq_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_1_7; - } - { - auto _gst_231 = dk_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_1_7; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j = 7 - k; + float a_qd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j))); + float a_kd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd + j))); + float a_dk[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk + j))); + float a_qa[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa + j))); + float a_ka[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka + j))); + float btr[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8 + j)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8 + j; + float r_248[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_248[0]; + unsigned int r_249[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_249[0] << 16); + unsigned int r_250[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_250[0] << 16); + float q_m = ((t >= clen) ? 0.0f : q_t); + float dqf = pool_f32[ew + 128 * j + d_lane]; + float dkf = pool_f32[ew + (1024 + 128 * j) + d_lane]; + float dgf = pool_f32[ew + (2048 + 128 * j) + d_lane]; + float bt_t = btr[0]; + float e_dg = approx_exp2(g_t - gc_0); + float e_off = approx_exp2(g_t - gn); + float dq2 = __fmaf_rn(a_qa[0], e_off, a_qd[0] * e_dg); + float dk2 = __fmaf_rn(a_ka[0], e_off, a_kd[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + a_dk[0]; + float qe = q_m * e_dg; + float kp = k_t * bt_t; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, a_qd[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg) * a_kd[0]); + float dg_o = __fmaf_rn(q_m, dq2, __fmaf_rn(dk2b - a_dk[0], k_t, cr)) + dgf; + s_run[0] = s_run[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j)), "f"(s_run[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff = t * qrs; + if (t < clen) { + { + auto _gst_248 = dq_base + qoff; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o; + } + { + auto _gst_249 = dk_base + qoff; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o; + } } } float tot = s_run[0]; + float dbp[8]; + tmem_ld_x8(&dbp[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); int b4 = lane_1 >> 4 & 1; int b3 = lane_1 >> 3 & 1; int b2 = lane_1 >> 2 & 1; float v4[4]; - float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; - float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; - float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; - float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; - float v2_1044[2]; - float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; - float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; - float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_250; + float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_251; + float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_252; + float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_253; + float v2[2]; + float send2_0 = ((b3 == 0) ? v4[2] : v4[0]); + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send2_0, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_254; + float send2_1 = ((b3 == 0) ? v4[3] : v4[1]); + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send2_1, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_255; + float send1 = ((b2 == 0) ? v2[1] : v2[0]); + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send1, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_256; + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_257; + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_258; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3346,412 +3195,148 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d if (lane_1 == 17) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(dt_bias + (hv * 128 + 32 * qd)))); } mbarrier_wait(epi_full_addr + (th) * 8, 1); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_272[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1049 = r_272[0]; - float gc_1050 = gn_1049 - 21.640425f; - int eb_1051 = 98304 + th * -32768; - int ew_1052 = eb_1051 >> 2; - int qbase_1053 = pool_f32_addr + (unsigned int)(eb_1051 + 12288 + dh_lane * 1024); - int sb8_1054 = 48 + 8 * th; - int gb8_1055 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1054 * 128); - int kb8_1056 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1054 * 128); - float dbp_1057[8]; - float _tmem_load_12[8]; - tmem_ld_x8(&_tmem_load_12[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float _tmem_load_13[8]; - tmem_ld_x8(&_tmem_load_13[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float _tmem_load_14[8]; - tmem_ld_x8(&_tmem_load_14[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float _tmem_load_15[8]; - tmem_ld_x8(&_tmem_load_15[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float _tmem_load_16[8]; - tmem_ld_x8(&_tmem_load_16[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_1058[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[0])), "=r"(*reinterpret_cast(&btv_1058[(0) + 1])), "=r"(*reinterpret_cast(&btv_1058[(0) + 2])), "=r"(*reinterpret_cast(&btv_1058[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[4])), "=r"(*reinterpret_cast(&btv_1058[(4) + 1])), "=r"(*reinterpret_cast(&btv_1058[(4) + 2])), "=r"(*reinterpret_cast(&btv_1058[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1059[1]; - s_run_1059[0] = 0.0f; - int t_0 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew_1052 + 896 + d_lane]; - float dkf_0 = pool_f32[ew_1052 + 1920 + d_lane]; - float dgf_0 = pool_f32[ew_1052 + 2944 + d_lane]; - float bt_t_0 = btv_1058[7]; - float e_off_0 = approx_exp2(g_t_0 - gn_1049); - float e_dg_0 = approx_exp2(g_t_0 - gc_1050); - float dq2_0 = __fmaf_rn(_tmem_load_12[7], e_off_0, _tmem_load_14[7] * e_dg_0); - float dk2_0 = __fmaf_rn(_tmem_load_13[7], e_off_0, _tmem_load_15[7] * e_dg_0); - dbp_1057[7] = dk2_0 * k_t_0; - float dk2b_0 = dk2_0 * bt_t_0; - float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; - s_run_1059[0] = s_run_1059[0] + dg_o_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_0 = t_0 * qrs; - if (t_0 < clen) { - { - auto _gst_241 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_0; - } - { - auto _gst_242 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_0; - } - } - int t_1 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew_1052 + 768 + d_lane]; - float dkf_1 = pool_f32[ew_1052 + 1792 + d_lane]; - float dgf_1 = pool_f32[ew_1052 + 2816 + d_lane]; - float bt_t_1 = btv_1058[6]; - float e_off_1 = approx_exp2(g_t_1 - gn_1049); - float e_dg_1 = approx_exp2(g_t_1 - gc_1050); - float dq2_1 = __fmaf_rn(_tmem_load_12[6], e_off_1, _tmem_load_14[6] * e_dg_1); - float dk2_1 = __fmaf_rn(_tmem_load_13[6], e_off_1, _tmem_load_15[6] * e_dg_1); - dbp_1057[6] = dk2_1 * k_t_1; - float dk2b_1 = dk2_1 * bt_t_1; - float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + _tmem_load_16[6]; - float dg_o_1 = __fmaf_rn(q_t_1_9, dq2_1, (dk2b_1 - _tmem_load_16[6]) * k_t_1) + dgf_1; - s_run_1059[0] = s_run_1059[0] + dg_o_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1 = t_1 * qrs; - if (t_1 < clen) { - { - auto _gst_243 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_1; - } - { - auto _gst_244 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_1; - } - } - int t_2 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew_1052 + 640 + d_lane]; - float dkf_2 = pool_f32[ew_1052 + 1664 + d_lane]; - float dgf_2 = pool_f32[ew_1052 + 2688 + d_lane]; - float bt_t_2 = btv_1058[5]; - float e_off_2 = approx_exp2(g_t_2 - gn_1049); - float e_dg_2 = approx_exp2(g_t_2 - gc_1050); - float dq2_2 = __fmaf_rn(_tmem_load_12[5], e_off_2, _tmem_load_14[5] * e_dg_2); - float dk2_2 = __fmaf_rn(_tmem_load_13[5], e_off_2, _tmem_load_15[5] * e_dg_2); - dbp_1057[5] = dk2_2 * k_t_2; - float dk2b_2 = dk2_2 * bt_t_2; - float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[5]; - float dg_o_2 = __fmaf_rn(q_t_1_10, dq2_2, (dk2b_2 - _tmem_load_16[5]) * k_t_2) + dgf_2; - s_run_1059[0] = s_run_1059[0] + dg_o_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_2 = t_2 * qrs; - if (t_2 < clen) { - { - auto _gst_245 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_2; - } - { - auto _gst_246 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_2; - } - } - int t_3 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew_1052 + 512 + d_lane]; - float dkf_3 = pool_f32[ew_1052 + 1536 + d_lane]; - float dgf_3 = pool_f32[ew_1052 + 2560 + d_lane]; - float bt_t_3 = btv_1058[4]; - float e_off_3 = approx_exp2(g_t_3 - gn_1049); - float e_dg_3 = approx_exp2(g_t_3 - gc_1050); - float dq2_3 = __fmaf_rn(_tmem_load_12[4], e_off_3, _tmem_load_14[4] * e_dg_3); - float dk2_3 = __fmaf_rn(_tmem_load_13[4], e_off_3, _tmem_load_15[4] * e_dg_3); - dbp_1057[4] = dk2_3 * k_t_3; - float dk2b_3 = dk2_3 * bt_t_3; - float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[4]; - float dg_o_3 = __fmaf_rn(q_t_1_11, dq2_3, (dk2b_3 - _tmem_load_16[4]) * k_t_3) + dgf_3; - s_run_1059[0] = s_run_1059[0] + dg_o_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_3 = t_3 * qrs; - if (t_3 < clen) { - { - auto _gst_247 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_3; - } - { - auto _gst_248 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_3; - } - } - int t_4 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew_1052 + 384 + d_lane]; - float dkf_4 = pool_f32[ew_1052 + 1408 + d_lane]; - float dgf_4 = pool_f32[ew_1052 + 2432 + d_lane]; - float bt_t_4 = btv_1058[3]; - float e_off_4 = approx_exp2(g_t_4 - gn_1049); - float e_dg_4 = approx_exp2(g_t_4 - gc_1050); - float dq2_4 = __fmaf_rn(_tmem_load_12[3], e_off_4, _tmem_load_14[3] * e_dg_4); - float dk2_4 = __fmaf_rn(_tmem_load_13[3], e_off_4, _tmem_load_15[3] * e_dg_4); - dbp_1057[3] = dk2_4 * k_t_4; - float dk2b_4 = dk2_4 * bt_t_4; - float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[3]; - float dg_o_4 = __fmaf_rn(q_t_1_12, dq2_4, (dk2b_4 - _tmem_load_16[3]) * k_t_4) + dgf_4; - s_run_1059[0] = s_run_1059[0] + dg_o_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_4 = t_4 * qrs; - if (t_4 < clen) { - { - auto _gst_249 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_4; - } - { - auto _gst_250 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_4; - } - } - int t_5 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew_1052 + 256 + d_lane]; - float dkf_5 = pool_f32[ew_1052 + 1280 + d_lane]; - float dgf_5 = pool_f32[ew_1052 + 2304 + d_lane]; - float bt_t_5 = btv_1058[2]; - float e_off_5 = approx_exp2(g_t_5 - gn_1049); - float e_dg_5 = approx_exp2(g_t_5 - gc_1050); - float dq2_5 = __fmaf_rn(_tmem_load_12[2], e_off_5, _tmem_load_14[2] * e_dg_5); - float dk2_5 = __fmaf_rn(_tmem_load_13[2], e_off_5, _tmem_load_15[2] * e_dg_5); - dbp_1057[2] = dk2_5 * k_t_5; - float dk2b_5 = dk2_5 * bt_t_5; - float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[2]; - float dg_o_5 = __fmaf_rn(q_t_1_13, dq2_5, (dk2b_5 - _tmem_load_16[2]) * k_t_5) + dgf_5; - s_run_1059[0] = s_run_1059[0] + dg_o_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_5 = t_5 * qrs; - if (t_5 < clen) { - { - auto _gst_251 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_5; - } - { - auto _gst_252 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_5; - } - } - int t_6 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew_1052 + 128 + d_lane]; - float dkf_6 = pool_f32[ew_1052 + 1152 + d_lane]; - float dgf_6 = pool_f32[ew_1052 + 2176 + d_lane]; - float bt_t_6 = btv_1058[1]; - float e_off_6 = approx_exp2(g_t_6 - gn_1049); - float e_dg_6 = approx_exp2(g_t_6 - gc_1050); - float dq2_6 = __fmaf_rn(_tmem_load_12[1], e_off_6, _tmem_load_14[1] * e_dg_6); - float dk2_6 = __fmaf_rn(_tmem_load_13[1], e_off_6, _tmem_load_15[1] * e_dg_6); - dbp_1057[1] = dk2_6 * k_t_6; - float dk2b_6 = dk2_6 * bt_t_6; - float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[1]; - float dg_o_6 = __fmaf_rn(q_t_1_14, dq2_6, (dk2b_6 - _tmem_load_16[1]) * k_t_6) + dgf_6; - s_run_1059[0] = s_run_1059[0] + dg_o_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_6 = t_6 * qrs; - if (t_6 < clen) { - { - auto _gst_253 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_6; - } - { - auto _gst_254 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_6; - } - } - int t_7 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew_1052 + d_lane]; - float dkf_7 = pool_f32[ew_1052 + 1024 + d_lane]; - float dgf_7 = pool_f32[ew_1052 + 2048 + d_lane]; - float bt_t_7 = btv_1058[0]; - float e_off_7 = approx_exp2(g_t_7 - gn_1049); - float e_dg_7 = approx_exp2(g_t_7 - gc_1050); - float dq2_7 = __fmaf_rn(_tmem_load_12[0], e_off_7, _tmem_load_14[0] * e_dg_7); - float dk2_7 = __fmaf_rn(_tmem_load_13[0], e_off_7, _tmem_load_15[0] * e_dg_7); - dbp_1057[0] = dk2_7 * k_t_7; - float dk2b_7 = dk2_7 * bt_t_7; - float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[0]; - float dg_o_7 = __fmaf_rn(q_t_1_15, dq2_7, (dk2b_7 - _tmem_load_16[0]) * k_t_7) + dgf_7; - s_run_1059[0] = s_run_1059[0] + dg_o_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_7 = t_7 * qrs; - if (t_7 < clen) { - { - auto _gst_255 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_7; - } - { - auto _gst_256 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_7; + float r_251[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_0 = r_251[0]; + float gc_1 = gn_0 - 21.640425f; + int eb_0 = 98304 + th * -32768; + int ew_0 = eb_0 >> 2; + int qbase_1 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); + int sb8_0 = 48 + 8 * th; + int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); + int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); + int c_qa_0 = 160 + 8 * th; + int c_ka_0 = 176 + 8 * th; + int c_qd_0 = 224 + 8 * th; + int c_kd_0 = 240 + 8 * th; + int c_dk_0 = 48 + 8 * th; + int c_bp_0 = c_ka_0; + float s_run_0[1]; + s_run_0[0] = 0.0f; + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_1 = 7 - k_1; + float a_qd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1))); + float a_kd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_0 + j_1))); + float a_dk_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_0 + j_1))); + float a_qa_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa_0 + j_1))); + float a_ka_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka_0 + j_1))); + float btr_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_0 + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_0 + j_1; + float r_252[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_252[0]; + unsigned int r_253[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_253[0] << 16); + unsigned int r_254[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1 = __uint_as_float(r_254[0] << 16); + float q_m_1 = ((t_1 >= clen) ? 0.0f : q_t_1); + float dqf_1 = pool_f32[ew_0 + 128 * j_1 + d_lane]; + float dkf_1 = pool_f32[ew_0 + (1024 + 128 * j_1) + d_lane]; + float dgf_1 = pool_f32[ew_0 + (2048 + 128 * j_1) + d_lane]; + float bt_t_1 = btr_1[0]; + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float e_off_1 = approx_exp2(g_t_1 - gn_0); + float dq2_1 = __fmaf_rn(a_qa_1[0], e_off_1, a_qd_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(a_ka_1[0], e_off_1, a_kd_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + a_dk_1[0]; + float qe_1 = q_m_1 * e_dg_1; + float kp_12 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_12; + float kbe_1 = (float)kb_1 * e_dg_1; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, a_qd_1[0], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_12 * e_dg_1) * a_kd_1[0]); + float dg_o_1 = __fmaf_rn(q_m_1, dq2_1, __fmaf_rn(dk2b_1 - a_dk_1[0], k_t_1, cr_1)) + dgf_1; + s_run_0[0] = s_run_0[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_1)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff_1 = t_1 * qrs; + if (t_1 < clen) { + { + auto _gst_259 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_1; + } + { + auto _gst_260 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_1; + } } } - float tot_1228 = s_run_1059[0]; - int b4_1229 = lane_1 >> 4 & 1; - int b3_1230 = lane_1 >> 3 & 1; - int b2_1231 = lane_1 >> 2 & 1; - float v4_1232[4]; - float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_257; - float send_1 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_258; - float send_2 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_259; - float send_3 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_260; - float v2_1237[2]; - float send_4 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_261; - float send_5 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_262; - float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_265; + float tot_0 = s_run_0[0]; + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int b4_0 = lane_1 >> 4 & 1; + int b3_0 = lane_1 >> 3 & 1; + int b2_0 = lane_1 >> 2 & 1; + float v4_0[4]; + float send_4 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_261; + float send_5 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_5, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_262; + float send_6 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_263; + float send_7 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_264; + float v2_16[2]; + float send2_2 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send2_2, 8); + v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_265; + float send2_3 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send2_3, 8); + v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_266; + float send1_0 = ((b2_0 == 0) ? v2_16[1] : v2_16[0]); + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send1_0, 4); + float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_267; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_268; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_269; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); float gacc[8]; @@ -3803,8 +3388,8 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_1228) : "memory"); + uint32_t _addr_270 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_270), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3822,8 +3407,8 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.fence::after_thread_sync;"); float t_other[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi = ((th == 1) ? tot_1228 : t_other[0]); - float t_lo = ((th == 1) ? t_other[0] : tot_1228); + float t_hi = ((th == 1) ? tot_0 : t_other[0]); + float t_lo = ((th == 1) ? t_other[0] : tot_0); float offset = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi); float sfx[8]; tmem_ld_x8(&sfx[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); @@ -3836,132 +3421,132 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_267 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_271 = make_float2(gacc[4], gacc[4]); float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&rp_u)); - float _abs_270; - asm("abs.f32 %0, %1;" : "=f"(_abs_270) : "f"(rp_z.x)); - float _neg_269; - asm("neg.f32 %0, %1;" : "=f"(_neg_269) : "f"(_abs_270)); - float _exp_268 = __nv_expf(_neg_269); - float _abs_273; - asm("abs.f32 %0, %1;" : "=f"(_abs_273) : "f"(rp_z.y)); - float _neg_272; - asm("neg.f32 %0, %1;" : "=f"(_neg_272) : "f"(_abs_273)); - float _exp_271 = __nv_expf(_neg_272); - float2 rp_e = make_float2(_exp_268, _exp_271); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_271), "l"(*(const unsigned long long*)&rp_u)); + float _abs_274; + asm("abs.f32 %0, %1;" : "=f"(_abs_274) : "f"(rp_z.x)); + float _neg_273; + asm("neg.f32 %0, %1;" : "=f"(_neg_273) : "f"(_abs_274)); + float _exp_272 = __nv_expf(_neg_273); + float _abs_277; + asm("abs.f32 %0, %1;" : "=f"(_abs_277) : "f"(rp_z.y)); + float _neg_276; + asm("neg.f32 %0, %1;" : "=f"(_neg_276) : "f"(_abs_277)); + float _exp_275 = __nv_expf(_neg_276); + float2 rp_e = make_float2(_exp_272, _exp_275); float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_274 = approx_rcp(rp_den.x); - float _rcp_275 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_276 = make_float2(_rcp_274, _rcp_275); + float _rcp_278 = approx_rcp(rp_den.x); + float _rcp_279 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_280 = make_float2(_rcp_278, _rcp_279); float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_276)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_280)); float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_277 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_281 = make_float2(gacc[5], gacc[5]); float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_281)); u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_278 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_282 = make_float2(gacc[4], gacc[4]); float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_278), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_281; - asm("abs.f32 %0, %1;" : "=f"(_abs_281) : "f"(rp_z_0.x)); - float _neg_280; - asm("neg.f32 %0, %1;" : "=f"(_neg_280) : "f"(_abs_281)); - float _exp_279 = __nv_expf(_neg_280); - float _abs_284; - asm("abs.f32 %0, %1;" : "=f"(_abs_284) : "f"(rp_z_0.y)); - float _neg_283; - asm("neg.f32 %0, %1;" : "=f"(_neg_283) : "f"(_abs_284)); - float _exp_282 = __nv_expf(_neg_283); - float2 rp_e_0 = make_float2(_exp_279, _exp_282); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_282), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_285; + asm("abs.f32 %0, %1;" : "=f"(_abs_285) : "f"(rp_z_0.x)); + float _neg_284; + asm("neg.f32 %0, %1;" : "=f"(_neg_284) : "f"(_abs_285)); + float _exp_283 = __nv_expf(_neg_284); + float _abs_288; + asm("abs.f32 %0, %1;" : "=f"(_abs_288) : "f"(rp_z_0.y)); + float _neg_287; + asm("neg.f32 %0, %1;" : "=f"(_neg_287) : "f"(_abs_288)); + float _exp_286 = __nv_expf(_neg_287); + float2 rp_e_0 = make_float2(_exp_283, _exp_286); float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_285 = approx_rcp(rp_den_0.x); - float _rcp_286 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_287 = make_float2(_rcp_285, _rcp_286); + float _rcp_289 = approx_rcp(rp_den_0.x); + float _rcp_290 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_291 = make_float2(_rcp_289, _rcp_290); float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_287)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_291)); float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_288 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_292 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_288)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_292)); u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_289 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_293 = make_float2(gacc[4], gacc[4]); float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_289), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_292; - asm("abs.f32 %0, %1;" : "=f"(_abs_292) : "f"(rp_z_1.x)); - float _neg_291; - asm("neg.f32 %0, %1;" : "=f"(_neg_291) : "f"(_abs_292)); - float _exp_290 = __nv_expf(_neg_291); - float _abs_295; - asm("abs.f32 %0, %1;" : "=f"(_abs_295) : "f"(rp_z_1.y)); - float _neg_294; - asm("neg.f32 %0, %1;" : "=f"(_neg_294) : "f"(_abs_295)); - float _exp_293 = __nv_expf(_neg_294); - float2 rp_e_1 = make_float2(_exp_290, _exp_293); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_293), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_296; + asm("abs.f32 %0, %1;" : "=f"(_abs_296) : "f"(rp_z_1.x)); + float _neg_295; + asm("neg.f32 %0, %1;" : "=f"(_neg_295) : "f"(_abs_296)); + float _exp_294 = __nv_expf(_neg_295); + float _abs_299; + asm("abs.f32 %0, %1;" : "=f"(_abs_299) : "f"(rp_z_1.y)); + float _neg_298; + asm("neg.f32 %0, %1;" : "=f"(_neg_298) : "f"(_abs_299)); + float _exp_297 = __nv_expf(_neg_298); + float2 rp_e_1 = make_float2(_exp_294, _exp_297); float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_296 = approx_rcp(rp_den_1.x); - float _rcp_297 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_298 = make_float2(_rcp_296, _rcp_297); + float _rcp_300 = approx_rcp(rp_den_1.x); + float _rcp_301 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_302 = make_float2(_rcp_300, _rcp_301); float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_298)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_302)); float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_299 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_303 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_299)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_303)); u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_300 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_304 = make_float2(gacc[4], gacc[4]); float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_303; - asm("abs.f32 %0, %1;" : "=f"(_abs_303) : "f"(rp_z_2.x)); - float _neg_302; - asm("neg.f32 %0, %1;" : "=f"(_neg_302) : "f"(_abs_303)); - float _exp_301 = __nv_expf(_neg_302); - float _abs_306; - asm("abs.f32 %0, %1;" : "=f"(_abs_306) : "f"(rp_z_2.y)); - float _neg_305; - asm("neg.f32 %0, %1;" : "=f"(_neg_305) : "f"(_abs_306)); - float _exp_304 = __nv_expf(_neg_305); - float2 rp_e_2 = make_float2(_exp_301, _exp_304); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_304), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_307; + asm("abs.f32 %0, %1;" : "=f"(_abs_307) : "f"(rp_z_2.x)); + float _neg_306; + asm("neg.f32 %0, %1;" : "=f"(_neg_306) : "f"(_abs_307)); + float _exp_305 = __nv_expf(_neg_306); + float _abs_310; + asm("abs.f32 %0, %1;" : "=f"(_abs_310) : "f"(rp_z_2.y)); + float _neg_309; + asm("neg.f32 %0, %1;" : "=f"(_neg_309) : "f"(_abs_310)); + float _exp_308 = __nv_expf(_neg_309); + float2 rp_e_2 = make_float2(_exp_305, _exp_308); float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_307 = approx_rcp(rp_den_2.x); - float _rcp_308 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_309 = make_float2(_rcp_307, _rcp_308); + float _rcp_311 = approx_rcp(rp_den_2.x); + float _rcp_312 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_313 = make_float2(_rcp_311, _rcp_312); float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_313)); float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_310 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_314 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_310)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_314)); u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -3976,54 +3561,54 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_311; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_311) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_311; + uint32_t _prmt_315; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_315; int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_312 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = pq_0[0]; + auto _gst_316 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = pq_0[0]; } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_313; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_313) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_313; + uint32_t _prmt_317; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_317; int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_314 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = pq_1[0]; + auto _gst_318 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = pq_1[0]; } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_315; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_315; + uint32_t _prmt_319; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_319) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_319; int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_316 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = pq_2[0]; + auto _gst_320 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = pq_2[0]; } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_317; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_317; + uint32_t _prmt_321; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_321) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_321; int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_318 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = pq_3[0]; + auto _gst_322 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast(_gst_322) + (0)) = pq_3[0]; } } int t_a_0 = 48 + 8 * th; @@ -4088,47 +3673,47 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_319; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_319) : "f"(s_a), "f"(nm_t)); + float _sub_323; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_323) : "f"(s_a), "f"(nm_t)); float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_319), "f"(pa)); - float _sub_320; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_320) : "f"(pa), "f"(nm_t)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_323), "f"(pa)); + float _sub_324; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_324) : "f"(pa), "f"(nm_t)); float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_320), "f"(s_a)); - float _abs_322; - asm("abs.f32 %0, %1;" : "=f"(_abs_322) : "f"(s_a)); - float _abs_323; - asm("abs.f32 %0, %1;" : "=f"(_abs_323) : "f"(pa)); - uint32_t _setp_321_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_321_bits) : "f"(_abs_322), "f"(_abs_323)); - bool _setp_321 = _setp_321_bits != 0; - float nm_err = ((_setp_321) ? nm_es : nm_ex); - float _add_324; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(c_a), "f"(nm_err)); - c_a = _add_324; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_324), "f"(s_a)); + float _abs_326; + asm("abs.f32 %0, %1;" : "=f"(_abs_326) : "f"(s_a)); + float _abs_327; + asm("abs.f32 %0, %1;" : "=f"(_abs_327) : "f"(pa)); + uint32_t _setp_325_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_325_bits) : "f"(_abs_326), "f"(_abs_327)); + bool _setp_325 = _setp_325_bits != 0; + float nm_err = ((_setp_325) ? nm_es : nm_ex); + float _add_328; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(c_a), "f"(nm_err)); + c_a = _add_328; s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_325; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_325) : "f"(s_b), "f"(nm_t_0)); + float _sub_329; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_329) : "f"(s_b), "f"(nm_t_0)); float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_325), "f"(pb)); - float _sub_326; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_326) : "f"(pb), "f"(nm_t_0)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_329), "f"(pb)); + float _sub_330; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_330) : "f"(pb), "f"(nm_t_0)); float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_326), "f"(s_b)); - float _abs_328; - asm("abs.f32 %0, %1;" : "=f"(_abs_328) : "f"(s_b)); - float _abs_329; - asm("abs.f32 %0, %1;" : "=f"(_abs_329) : "f"(pb)); - uint32_t _setp_327_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_327_bits) : "f"(_abs_328), "f"(_abs_329)); - bool _setp_327 = _setp_327_bits != 0; - float nm_err_0 = ((_setp_327) ? nm_es_0 : nm_ex_0); - float _add_330; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_330; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_330), "f"(s_b)); + float _abs_332; + asm("abs.f32 %0, %1;" : "=f"(_abs_332) : "f"(s_b)); + float _abs_333; + asm("abs.f32 %0, %1;" : "=f"(_abs_333) : "f"(pb)); + uint32_t _setp_331_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_331_bits) : "f"(_abs_332), "f"(_abs_333)); + bool _setp_331 = _setp_331_bits != 0; + float nm_err_0 = ((_setp_331) ? nm_es_0 : nm_ex_0); + float _add_334; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_334; s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4152,132 +3737,132 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_331 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_335 = make_float2(gacc[4], gacc[4]); float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_331), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_334; - asm("abs.f32 %0, %1;" : "=f"(_abs_334) : "f"(rp_z_3.x)); - float _neg_333; - asm("neg.f32 %0, %1;" : "=f"(_neg_333) : "f"(_abs_334)); - float _exp_332 = __nv_expf(_neg_333); - float _abs_337; - asm("abs.f32 %0, %1;" : "=f"(_abs_337) : "f"(rp_z_3.y)); - float _neg_336; - asm("neg.f32 %0, %1;" : "=f"(_neg_336) : "f"(_abs_337)); - float _exp_335 = __nv_expf(_neg_336); - float2 rp_e_3 = make_float2(_exp_332, _exp_335); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_335), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_338; + asm("abs.f32 %0, %1;" : "=f"(_abs_338) : "f"(rp_z_3.x)); + float _neg_337; + asm("neg.f32 %0, %1;" : "=f"(_neg_337) : "f"(_abs_338)); + float _exp_336 = __nv_expf(_neg_337); + float _abs_341; + asm("abs.f32 %0, %1;" : "=f"(_abs_341) : "f"(rp_z_3.y)); + float _neg_340; + asm("neg.f32 %0, %1;" : "=f"(_neg_340) : "f"(_abs_341)); + float _exp_339 = __nv_expf(_neg_340); + float2 rp_e_3 = make_float2(_exp_336, _exp_339); float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_338 = approx_rcp(rp_den_3.x); - float _rcp_339 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_340 = make_float2(_rcp_338, _rcp_339); + float _rcp_342 = approx_rcp(rp_den_3.x); + float _rcp_343 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_344 = make_float2(_rcp_342, _rcp_343); float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_340)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_344)); float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_341 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_345 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_345)); u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_342 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_346 = make_float2(gacc[4], gacc[4]); float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_342), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_345; - asm("abs.f32 %0, %1;" : "=f"(_abs_345) : "f"(rp_z_4.x)); - float _neg_344; - asm("neg.f32 %0, %1;" : "=f"(_neg_344) : "f"(_abs_345)); - float _exp_343 = __nv_expf(_neg_344); - float _abs_348; - asm("abs.f32 %0, %1;" : "=f"(_abs_348) : "f"(rp_z_4.y)); - float _neg_347; - asm("neg.f32 %0, %1;" : "=f"(_neg_347) : "f"(_abs_348)); - float _exp_346 = __nv_expf(_neg_347); - float2 rp_e_4 = make_float2(_exp_343, _exp_346); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_346), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_349; + asm("abs.f32 %0, %1;" : "=f"(_abs_349) : "f"(rp_z_4.x)); + float _neg_348; + asm("neg.f32 %0, %1;" : "=f"(_neg_348) : "f"(_abs_349)); + float _exp_347 = __nv_expf(_neg_348); + float _abs_352; + asm("abs.f32 %0, %1;" : "=f"(_abs_352) : "f"(rp_z_4.y)); + float _neg_351; + asm("neg.f32 %0, %1;" : "=f"(_neg_351) : "f"(_abs_352)); + float _exp_350 = __nv_expf(_neg_351); + float2 rp_e_4 = make_float2(_exp_347, _exp_350); float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_349 = approx_rcp(rp_den_4.x); - float _rcp_350 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_351 = make_float2(_rcp_349, _rcp_350); + float _rcp_353 = approx_rcp(rp_den_4.x); + float _rcp_354 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_355 = make_float2(_rcp_353, _rcp_354); float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_351)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_355)); float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_352 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_356 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_352)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_356)); u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_353 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_357 = make_float2(gacc[4], gacc[4]); float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_356; - asm("abs.f32 %0, %1;" : "=f"(_abs_356) : "f"(rp_z_5.x)); - float _neg_355; - asm("neg.f32 %0, %1;" : "=f"(_neg_355) : "f"(_abs_356)); - float _exp_354 = __nv_expf(_neg_355); - float _abs_359; - asm("abs.f32 %0, %1;" : "=f"(_abs_359) : "f"(rp_z_5.y)); - float _neg_358; - asm("neg.f32 %0, %1;" : "=f"(_neg_358) : "f"(_abs_359)); - float _exp_357 = __nv_expf(_neg_358); - float2 rp_e_5 = make_float2(_exp_354, _exp_357); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_357), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_360; + asm("abs.f32 %0, %1;" : "=f"(_abs_360) : "f"(rp_z_5.x)); + float _neg_359; + asm("neg.f32 %0, %1;" : "=f"(_neg_359) : "f"(_abs_360)); + float _exp_358 = __nv_expf(_neg_359); + float _abs_363; + asm("abs.f32 %0, %1;" : "=f"(_abs_363) : "f"(rp_z_5.y)); + float _neg_362; + asm("neg.f32 %0, %1;" : "=f"(_neg_362) : "f"(_abs_363)); + float _exp_361 = __nv_expf(_neg_362); + float2 rp_e_5 = make_float2(_exp_358, _exp_361); float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_360 = approx_rcp(rp_den_5.x); - float _rcp_361 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_362 = make_float2(_rcp_360, _rcp_361); + float _rcp_364 = approx_rcp(rp_den_5.x); + float _rcp_365 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_366 = make_float2(_rcp_364, _rcp_365); float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_362)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_366)); float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_363 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_367 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_363)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_367)); u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_364 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_368 = make_float2(gacc[4], gacc[4]); float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_364), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_367; - asm("abs.f32 %0, %1;" : "=f"(_abs_367) : "f"(rp_z_6.x)); - float _neg_366; - asm("neg.f32 %0, %1;" : "=f"(_neg_366) : "f"(_abs_367)); - float _exp_365 = __nv_expf(_neg_366); - float _abs_370; - asm("abs.f32 %0, %1;" : "=f"(_abs_370) : "f"(rp_z_6.y)); - float _neg_369; - asm("neg.f32 %0, %1;" : "=f"(_neg_369) : "f"(_abs_370)); - float _exp_368 = __nv_expf(_neg_369); - float2 rp_e_6 = make_float2(_exp_365, _exp_368); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_368), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_371; + asm("abs.f32 %0, %1;" : "=f"(_abs_371) : "f"(rp_z_6.x)); + float _neg_370; + asm("neg.f32 %0, %1;" : "=f"(_neg_370) : "f"(_abs_371)); + float _exp_369 = __nv_expf(_neg_370); + float _abs_374; + asm("abs.f32 %0, %1;" : "=f"(_abs_374) : "f"(rp_z_6.y)); + float _neg_373; + asm("neg.f32 %0, %1;" : "=f"(_neg_373) : "f"(_abs_374)); + float _exp_372 = __nv_expf(_neg_373); + float2 rp_e_6 = make_float2(_exp_369, _exp_372); float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_371 = approx_rcp(rp_den_6.x); - float _rcp_372 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_373 = make_float2(_rcp_371, _rcp_372); + float _rcp_375 = approx_rcp(rp_den_6.x); + float _rcp_376 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_377 = make_float2(_rcp_375, _rcp_376); float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_373)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_377)); float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_374 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_378 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_374)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_378)); u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4292,54 +3877,54 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_375; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_375) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_375; + uint32_t _prmt_379; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_379; int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_376 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_376)); - *(reinterpret_cast(_gst_376) + (0)) = pq_4[0]; + auto _gst_380 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = pq_4[0]; } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_377; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_377) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_377; + uint32_t _prmt_381; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_381; int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_378 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_378)); - *(reinterpret_cast(_gst_378) + (0)) = pq_5[0]; + auto _gst_382 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = pq_5[0]; } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_379; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_379; + uint32_t _prmt_383; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_383) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_383; int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_380 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_380)); - *(reinterpret_cast(_gst_380) + (0)) = pq_6[0]; + auto _gst_384 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = pq_6[0]; } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_381; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_381; + uint32_t _prmt_385; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_385) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_385; int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_382 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_382)); - *(reinterpret_cast(_gst_382) + (0)) = pq_7[0]; + auto _gst_386 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_386)); + *(reinterpret_cast(_gst_386) + (0)) = pq_7[0]; } } int t_a_8 = 32 + 8 * th; @@ -4404,47 +3989,47 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_383; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_383) : "f"(s_a_0), "f"(nm_t_1)); + float _sub_387; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_387) : "f"(s_a_0), "f"(nm_t_1)); float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_383), "f"(pa_0)); - float _sub_384; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_384) : "f"(pa_0), "f"(nm_t_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_387), "f"(pa_0)); + float _sub_388; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_388) : "f"(pa_0), "f"(nm_t_1)); float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_384), "f"(s_a_0)); - float _abs_386; - asm("abs.f32 %0, %1;" : "=f"(_abs_386) : "f"(s_a_0)); - float _abs_387; - asm("abs.f32 %0, %1;" : "=f"(_abs_387) : "f"(pa_0)); - uint32_t _setp_385_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_385_bits) : "f"(_abs_386), "f"(_abs_387)); - bool _setp_385 = _setp_385_bits != 0; - float nm_err_1 = ((_setp_385) ? nm_es_1 : nm_ex_1); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_388), "f"(s_a_0)); + float _abs_390; + asm("abs.f32 %0, %1;" : "=f"(_abs_390) : "f"(s_a_0)); + float _abs_391; + asm("abs.f32 %0, %1;" : "=f"(_abs_391) : "f"(pa_0)); + uint32_t _setp_389_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_389_bits) : "f"(_abs_390), "f"(_abs_391)); + bool _setp_389 = _setp_389_bits != 0; + float nm_err_1 = ((_setp_389) ? nm_es_1 : nm_ex_1); + float _add_392; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_392; s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_389; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_389) : "f"(s_b_0), "f"(nm_t_2)); + float _sub_393; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_393) : "f"(s_b_0), "f"(nm_t_2)); float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_389), "f"(pb_0)); - float _sub_390; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_390) : "f"(pb_0), "f"(nm_t_2)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_393), "f"(pb_0)); + float _sub_394; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_394) : "f"(pb_0), "f"(nm_t_2)); float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_390), "f"(s_b_0)); - float _abs_392; - asm("abs.f32 %0, %1;" : "=f"(_abs_392) : "f"(s_b_0)); - float _abs_393; - asm("abs.f32 %0, %1;" : "=f"(_abs_393) : "f"(pb_0)); - uint32_t _setp_391_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_391_bits) : "f"(_abs_392), "f"(_abs_393)); - bool _setp_391 = _setp_391_bits != 0; - float nm_err_2 = ((_setp_391) ? nm_es_2 : nm_ex_2); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_394; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_394), "f"(s_b_0)); + float _abs_396; + asm("abs.f32 %0, %1;" : "=f"(_abs_396) : "f"(s_b_0)); + float _abs_397; + asm("abs.f32 %0, %1;" : "=f"(_abs_397) : "f"(pb_0)); + uint32_t _setp_395_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_395_bits) : "f"(_abs_396), "f"(_abs_397)); + bool _setp_395 = _setp_395_bits != 0; + float nm_err_2 = ((_setp_395) ? nm_es_2 : nm_ex_2); + float _add_398; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_398; s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4452,8 +4037,8 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_395 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_395), "f"(off_acc[0]) : "memory"); + uint32_t _addr_399 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_399), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4479,9 +4064,9 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1292 = r_297[0]; + float r_255[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1292 = r_255[0]; float gc_1293 = gn_1292 - 21.640425f; int eb_1294 = 81920 + th * -32768; int ew_1295 = eb_1294 >> 2; @@ -4510,348 +4095,388 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1302[1]; s_run_1302[0] = 0.0f; - int t_8 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_1295 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_1295 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_1295 + 2944 + d_lane]; - float bt_t_8 = btv_1301[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_1292); - float e_dg_8 = approx_exp2(g_t_8 - gc_1293); - float dq2_8 = __fmaf_rn(_tmem_load_20[7], e_off_8, _tmem_load_22[7] * e_dg_8); - float dk2_8 = __fmaf_rn(_tmem_load_21[7], e_off_8, _tmem_load_23[7] * e_dg_8); - dbp_1300[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_24[7]; - float dg_o_8 = __fmaf_rn(q_t_1_16, dq2_8, (dk2b_8 - _tmem_load_24[7]) * k_t_8) + dgf_8; - s_run_1302[0] = s_run_1302[0] + dg_o_8; + int t_0 = 16 + 8 * th + 7; + float r_256[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_0 = r_256[0]; + unsigned int r_257[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_0 = __uint_as_float(r_257[0] << 16); + unsigned int r_258[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_0 = __uint_as_float(r_258[0] << 16); + float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); + float dqf_0 = pool_f32[ew_1295 + 896 + d_lane]; + float dkf_0 = pool_f32[ew_1295 + 1920 + d_lane]; + float dgf_0 = pool_f32[ew_1295 + 2944 + d_lane]; + float bt_t_0 = btv_1301[7]; + float e_off_0 = approx_exp2(g_t_0 - gn_1292); + float e_dg_0 = approx_exp2(g_t_0 - gc_1293); + float dq2_0 = __fmaf_rn(_tmem_load_20[7], e_off_0, _tmem_load_22[7] * e_dg_0); + float dk2_0 = __fmaf_rn(_tmem_load_21[7], e_off_0, _tmem_load_23[7] * e_dg_0); + dbp_1300[7] = dk2_0 * k_t_0; + float dk2b_0 = dk2_0 * bt_t_0; + float dq_o_0 = dq2_0 + dqf_0; + float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_24[7]; + float qe_2 = q_t_1_0 * e_dg_0; + float kp_13 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_13; + float kbe_2 = (float)kb_2 * e_dg_0; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_13 * e_dg_0) * _tmem_load_23[7]); + float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_24[7], k_t_0, cr_2)) + dgf_0; + s_run_1302[0] = s_run_1302[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_8 = t_8 * qrs; - if (t_8 < clen) { + int qoff_0 = t_0 * qrs; + if (t_0 < clen) { { - auto _gst_396 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_396)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_396) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_400 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_400)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_397 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_397)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_397) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_401 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_401)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_0; } } - int t_9 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_1295 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_1295 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_1295 + 2816 + d_lane]; - float bt_t_9 = btv_1301[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_1292); - float e_dg_9 = approx_exp2(g_t_9 - gc_1293); - float dq2_9 = __fmaf_rn(_tmem_load_20[6], e_off_9, _tmem_load_22[6] * e_dg_9); - float dk2_9 = __fmaf_rn(_tmem_load_21[6], e_off_9, _tmem_load_23[6] * e_dg_9); - dbp_1300[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[6]; - float dg_o_9 = __fmaf_rn(q_t_1_17, dq2_9, (dk2b_9 - _tmem_load_24[6]) * k_t_9) + dgf_9; - s_run_1302[0] = s_run_1302[0] + dg_o_9; + int t_1_1 = 16 + 8 * th + 6; + float r_259[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1_1 = r_259[0]; + unsigned int r_260[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1_1 = __uint_as_float(r_260[0] << 16); + unsigned int r_261[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1_1 = __uint_as_float(r_261[0] << 16); + float q_t_1_1_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_1); + float dqf_1_1 = pool_f32[ew_1295 + 768 + d_lane]; + float dkf_1_1 = pool_f32[ew_1295 + 1792 + d_lane]; + float dgf_1_1 = pool_f32[ew_1295 + 2816 + d_lane]; + float bt_t_1_1 = btv_1301[6]; + float e_off_1_1 = approx_exp2(g_t_1_1 - gn_1292); + float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_1293); + float dq2_1_1 = __fmaf_rn(_tmem_load_20[6], e_off_1_1, _tmem_load_22[6] * e_dg_1_1); + float dk2_1_1 = __fmaf_rn(_tmem_load_21[6], e_off_1_1, _tmem_load_23[6] * e_dg_1_1); + dbp_1300[6] = dk2_1_1 * k_t_1_1; + float dk2b_1_1 = dk2_1_1 * bt_t_1_1; + float dq_o_1_1 = dq2_1_1 + dqf_1_1; + float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_24[6]; + float qe_0 = q_t_1_1_1 * e_dg_1_1; + float kp_12_1 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12_1; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12_1 * e_dg_1_1) * _tmem_load_23[6]); + float dg_o_1_1 = __fmaf_rn(q_t_1_1_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_24[6], k_t_1_1, cr_0)) + dgf_1_1; + s_run_1302[0] = s_run_1302[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_9 = t_9 * qrs; - if (t_9 < clen) { + int qoff_1_1 = t_1_1 * qrs; + if (t_1_1 < clen) { { - auto _gst_398 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_398)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_398) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_402 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_402)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_1_1; } { - auto _gst_399 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_399)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_399) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_403 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_403)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_1_1; } } - int t_10 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_1295 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_1295 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_1295 + 2688 + d_lane]; - float bt_t_10 = btv_1301[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_1292); - float e_dg_10 = approx_exp2(g_t_10 - gc_1293); - float dq2_10 = __fmaf_rn(_tmem_load_20[5], e_off_10, _tmem_load_22[5] * e_dg_10); - float dk2_10 = __fmaf_rn(_tmem_load_21[5], e_off_10, _tmem_load_23[5] * e_dg_10); - dbp_1300[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[5]; - float dg_o_10 = __fmaf_rn(q_t_1_18, dq2_10, (dk2b_10 - _tmem_load_24[5]) * k_t_10) + dgf_10; - s_run_1302[0] = s_run_1302[0] + dg_o_10; + int t_2 = 16 + 8 * th + 5; + float r_262[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_262[0]; + unsigned int r_263[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_263[0] << 16); + unsigned int r_264[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_264[0] << 16); + float q_t_1_2 = ((t_2 >= clen) ? 0.0f : q_t_2); + float dqf_2 = pool_f32[ew_1295 + 640 + d_lane]; + float dkf_2 = pool_f32[ew_1295 + 1664 + d_lane]; + float dgf_2 = pool_f32[ew_1295 + 2688 + d_lane]; + float bt_t_2 = btv_1301[5]; + float e_off_2 = approx_exp2(g_t_2 - gn_1292); + float e_dg_2 = approx_exp2(g_t_2 - gc_1293); + float dq2_2 = __fmaf_rn(_tmem_load_20[5], e_off_2, _tmem_load_22[5] * e_dg_2); + float dk2_2 = __fmaf_rn(_tmem_load_21[5], e_off_2, _tmem_load_23[5] * e_dg_2); + dbp_1300[5] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_24[5]; + float qe_1_1 = q_t_1_2 * e_dg_2; + float kp_13_1 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1_1 = (__nv_bfloat16)kp_13_1; + float kbe_1_1 = (float)kb_1_1 * e_dg_2; + float cr_1_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1_1) & 4294959104u) - qe_1_1, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_1_1) & 4294959104u) - kp_13_1 * e_dg_2) * _tmem_load_23[5]); + float dg_o_2 = __fmaf_rn(q_t_1_2, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_24[5], k_t_2, cr_1_1)) + dgf_2; + s_run_1302[0] = s_run_1302[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_10 = t_10 * qrs; - if (t_10 < clen) { + int qoff_2 = t_2 * qrs; + if (t_2 < clen) { { - auto _gst_400 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_400)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_404 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_404)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_401 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_401)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_405 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_405)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_2; } } - int t_11 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_1295 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_1295 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_1295 + 2560 + d_lane]; - float bt_t_11 = btv_1301[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_1292); - float e_dg_11 = approx_exp2(g_t_11 - gc_1293); - float dq2_11 = __fmaf_rn(_tmem_load_20[4], e_off_11, _tmem_load_22[4] * e_dg_11); - float dk2_11 = __fmaf_rn(_tmem_load_21[4], e_off_11, _tmem_load_23[4] * e_dg_11); - dbp_1300[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[4]; - float dg_o_11 = __fmaf_rn(q_t_1_19, dq2_11, (dk2b_11 - _tmem_load_24[4]) * k_t_11) + dgf_11; - s_run_1302[0] = s_run_1302[0] + dg_o_11; + int t_3 = 16 + 8 * th + 4; + float r_265[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_3 = r_265[0]; + unsigned int r_266[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_3 = __uint_as_float(r_266[0] << 16); + unsigned int r_267[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_3 = __uint_as_float(r_267[0] << 16); + float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); + float dqf_3 = pool_f32[ew_1295 + 512 + d_lane]; + float dkf_3 = pool_f32[ew_1295 + 1536 + d_lane]; + float dgf_3 = pool_f32[ew_1295 + 2560 + d_lane]; + float bt_t_3 = btv_1301[4]; + float e_off_3 = approx_exp2(g_t_3 - gn_1292); + float e_dg_3 = approx_exp2(g_t_3 - gc_1293); + float dq2_3 = __fmaf_rn(_tmem_load_20[4], e_off_3, _tmem_load_22[4] * e_dg_3); + float dk2_3 = __fmaf_rn(_tmem_load_21[4], e_off_3, _tmem_load_23[4] * e_dg_3); + dbp_1300[4] = dk2_3 * k_t_3; + float dk2b_3 = dk2_3 * bt_t_3; + float dq_o_3 = dq2_3 + dqf_3; + float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_24[4]; + float qe_2_1 = q_t_1_3 * e_dg_3; + float kp_14 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2_1 = (__nv_bfloat16)kp_14; + float kbe_2_1 = (float)kb_2_1 * e_dg_3; + float cr_2_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_2_1) & 4294959104u) - qe_2_1, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_2_1) & 4294959104u) - kp_14 * e_dg_3) * _tmem_load_23[4]); + float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_24[4], k_t_3, cr_2_1)) + dgf_3; + s_run_1302[0] = s_run_1302[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_11 = t_11 * qrs; - if (t_11 < clen) { + int qoff_3 = t_3 * qrs; + if (t_3 < clen) { { - auto _gst_402 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_402)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_406 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_406)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_403 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_403)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_407 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_407)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_3; } } - int t_12 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_1295 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_1295 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_1295 + 2432 + d_lane]; - float bt_t_12 = btv_1301[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_1292); - float e_dg_12 = approx_exp2(g_t_12 - gc_1293); - float dq2_12 = __fmaf_rn(_tmem_load_20[3], e_off_12, _tmem_load_22[3] * e_dg_12); - float dk2_12 = __fmaf_rn(_tmem_load_21[3], e_off_12, _tmem_load_23[3] * e_dg_12); - dbp_1300[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[3]; - float dg_o_12 = __fmaf_rn(q_t_1_20, dq2_12, (dk2b_12 - _tmem_load_24[3]) * k_t_12) + dgf_12; - s_run_1302[0] = s_run_1302[0] + dg_o_12; + int t_4 = 16 + 8 * th + 3; + float r_268[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_4 = r_268[0]; + unsigned int r_269[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_4 = __uint_as_float(r_269[0] << 16); + unsigned int r_270[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_4 = __uint_as_float(r_270[0] << 16); + float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); + float dqf_4 = pool_f32[ew_1295 + 384 + d_lane]; + float dkf_4 = pool_f32[ew_1295 + 1408 + d_lane]; + float dgf_4 = pool_f32[ew_1295 + 2432 + d_lane]; + float bt_t_4 = btv_1301[3]; + float e_off_4 = approx_exp2(g_t_4 - gn_1292); + float e_dg_4 = approx_exp2(g_t_4 - gc_1293); + float dq2_4 = __fmaf_rn(_tmem_load_20[3], e_off_4, _tmem_load_22[3] * e_dg_4); + float dk2_4 = __fmaf_rn(_tmem_load_21[3], e_off_4, _tmem_load_23[3] * e_dg_4); + dbp_1300[3] = dk2_4 * k_t_4; + float dk2b_4 = dk2_4 * bt_t_4; + float dq_o_4 = dq2_4 + dqf_4; + float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_24[3]; + float qe_3 = q_t_1_4 * e_dg_4; + float kp_15 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_4) * _tmem_load_23[3]); + float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_24[3], k_t_4, cr_3)) + dgf_4; + s_run_1302[0] = s_run_1302[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_12 = t_12 * qrs; - if (t_12 < clen) { + int qoff_4 = t_4 * qrs; + if (t_4 < clen) { { - auto _gst_404 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_404)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_408 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_408)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_405 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_405)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_409 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_409)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_4; } } - int t_13 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_1295 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_1295 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_1295 + 2304 + d_lane]; - float bt_t_13 = btv_1301[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_1292); - float e_dg_13 = approx_exp2(g_t_13 - gc_1293); - float dq2_13 = __fmaf_rn(_tmem_load_20[2], e_off_13, _tmem_load_22[2] * e_dg_13); - float dk2_13 = __fmaf_rn(_tmem_load_21[2], e_off_13, _tmem_load_23[2] * e_dg_13); - dbp_1300[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[2]; - float dg_o_13 = __fmaf_rn(q_t_1_21, dq2_13, (dk2b_13 - _tmem_load_24[2]) * k_t_13) + dgf_13; - s_run_1302[0] = s_run_1302[0] + dg_o_13; + int t_5 = 16 + 8 * th + 2; + float r_271[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_5 = r_271[0]; + unsigned int r_272[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_5 = __uint_as_float(r_272[0] << 16); + unsigned int r_273[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_5 = __uint_as_float(r_273[0] << 16); + float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); + float dqf_5 = pool_f32[ew_1295 + 256 + d_lane]; + float dkf_5 = pool_f32[ew_1295 + 1280 + d_lane]; + float dgf_5 = pool_f32[ew_1295 + 2304 + d_lane]; + float bt_t_5 = btv_1301[2]; + float e_off_5 = approx_exp2(g_t_5 - gn_1292); + float e_dg_5 = approx_exp2(g_t_5 - gc_1293); + float dq2_5 = __fmaf_rn(_tmem_load_20[2], e_off_5, _tmem_load_22[2] * e_dg_5); + float dk2_5 = __fmaf_rn(_tmem_load_21[2], e_off_5, _tmem_load_23[2] * e_dg_5); + dbp_1300[2] = dk2_5 * k_t_5; + float dk2b_5 = dk2_5 * bt_t_5; + float dq_o_5 = dq2_5 + dqf_5; + float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_24[2]; + float qe_4 = q_t_1_5 * e_dg_5; + float kp_16 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_5) * _tmem_load_23[2]); + float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_24[2], k_t_5, cr_4)) + dgf_5; + s_run_1302[0] = s_run_1302[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_13 = t_13 * qrs; - if (t_13 < clen) { + int qoff_5 = t_5 * qrs; + if (t_5 < clen) { { - auto _gst_406 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_406)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_410 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_410)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_407 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_407)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_411 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_411)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_5; } } - int t_14 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_1295 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_1295 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_1295 + 2176 + d_lane]; - float bt_t_14 = btv_1301[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_1292); - float e_dg_14 = approx_exp2(g_t_14 - gc_1293); - float dq2_14 = __fmaf_rn(_tmem_load_20[1], e_off_14, _tmem_load_22[1] * e_dg_14); - float dk2_14 = __fmaf_rn(_tmem_load_21[1], e_off_14, _tmem_load_23[1] * e_dg_14); - dbp_1300[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[1]; - float dg_o_14 = __fmaf_rn(q_t_1_22, dq2_14, (dk2b_14 - _tmem_load_24[1]) * k_t_14) + dgf_14; - s_run_1302[0] = s_run_1302[0] + dg_o_14; + int t_6 = 16 + 8 * th + 1; + float r_274[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_6 = r_274[0]; + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_6 = __uint_as_float(r_275[0] << 16); + unsigned int r_276[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_6 = __uint_as_float(r_276[0] << 16); + float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); + float dqf_6 = pool_f32[ew_1295 + 128 + d_lane]; + float dkf_6 = pool_f32[ew_1295 + 1152 + d_lane]; + float dgf_6 = pool_f32[ew_1295 + 2176 + d_lane]; + float bt_t_6 = btv_1301[1]; + float e_off_6 = approx_exp2(g_t_6 - gn_1292); + float e_dg_6 = approx_exp2(g_t_6 - gc_1293); + float dq2_6 = __fmaf_rn(_tmem_load_20[1], e_off_6, _tmem_load_22[1] * e_dg_6); + float dk2_6 = __fmaf_rn(_tmem_load_21[1], e_off_6, _tmem_load_23[1] * e_dg_6); + dbp_1300[1] = dk2_6 * k_t_6; + float dk2b_6 = dk2_6 * bt_t_6; + float dq_o_6 = dq2_6 + dqf_6; + float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_24[1]; + float qe_5 = q_t_1_6 * e_dg_6; + float kp_17 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_6) * _tmem_load_23[1]); + float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_24[1], k_t_6, cr_5)) + dgf_6; + s_run_1302[0] = s_run_1302[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_14 = t_14 * qrs; - if (t_14 < clen) { + int qoff_6 = t_6 * qrs; + if (t_6 < clen) { { - auto _gst_408 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_408)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_412 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_412)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_412) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_409 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_409)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_413 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_413)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_413) + (0)) = (__nv_bfloat16)dk_o_6; } } - int t_15 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_1295 + d_lane]; - float dkf_15 = pool_f32[ew_1295 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_1295 + 2048 + d_lane]; - float bt_t_15 = btv_1301[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_1292); - float e_dg_15 = approx_exp2(g_t_15 - gc_1293); - float dq2_15 = __fmaf_rn(_tmem_load_20[0], e_off_15, _tmem_load_22[0] * e_dg_15); - float dk2_15 = __fmaf_rn(_tmem_load_21[0], e_off_15, _tmem_load_23[0] * e_dg_15); - dbp_1300[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[0]; - float dg_o_15 = __fmaf_rn(q_t_1_23, dq2_15, (dk2b_15 - _tmem_load_24[0]) * k_t_15) + dgf_15; - s_run_1302[0] = s_run_1302[0] + dg_o_15; + int t_7 = 16 + 8 * th; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_7 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_7 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_7 = __uint_as_float(r_279[0] << 16); + float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); + float dqf_7 = pool_f32[ew_1295 + d_lane]; + float dkf_7 = pool_f32[ew_1295 + 1024 + d_lane]; + float dgf_7 = pool_f32[ew_1295 + 2048 + d_lane]; + float bt_t_7 = btv_1301[0]; + float e_off_7 = approx_exp2(g_t_7 - gn_1292); + float e_dg_7 = approx_exp2(g_t_7 - gc_1293); + float dq2_7 = __fmaf_rn(_tmem_load_20[0], e_off_7, _tmem_load_22[0] * e_dg_7); + float dk2_7 = __fmaf_rn(_tmem_load_21[0], e_off_7, _tmem_load_23[0] * e_dg_7); + dbp_1300[0] = dk2_7 * k_t_7; + float dk2b_7 = dk2_7 * bt_t_7; + float dq_o_7 = dq2_7 + dqf_7; + float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_24[0]; + float qe_6 = q_t_1_7 * e_dg_7; + float kp_18 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_7) * _tmem_load_23[0]); + float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_24[0], k_t_7, cr_6)) + dgf_7; + s_run_1302[0] = s_run_1302[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_15 = t_15 * qrs; - if (t_15 < clen) { + int qoff_7 = t_7 * qrs; + if (t_7 < clen) { { - auto _gst_410 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_410)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_414 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_414)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_414) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_411 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_411)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_415 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_415)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_415) + (0)) = (__nv_bfloat16)dk_o_7; } } float tot_1471 = s_run_1302[0]; @@ -4859,40 +4484,40 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d int b3_1473 = lane_1 >> 3 & 1; int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; - float send_6 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_412 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_412; - float send_7 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_413; - float send_8 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_414 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_414; - float send_9 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_415; + float send_8 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); + float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_416; + float send_9 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_417; + float send_10 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); + float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_418; + float send_11 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, send_11, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_419; float v2_1480[2]; - float send_10 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_416; - float send_11 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_417; + float send_12 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); + float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_420; + float send_13 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, send_13, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_421; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_418; - float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_419; - float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_420; + float _shfl_xor_sync_422 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_422; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_423; + float _shfl_xor_sync_424 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_424; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_421 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_421), "f"(tot_1471) : "memory"); + uint32_t _addr_425 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_425), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4924,132 +4549,132 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_422 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_426 = make_float2(gacc[4], gacc[4]); float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_422), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_425; - asm("abs.f32 %0, %1;" : "=f"(_abs_425) : "f"(rp_z_7.x)); - float _neg_424; - asm("neg.f32 %0, %1;" : "=f"(_neg_424) : "f"(_abs_425)); - float _exp_423 = __nv_expf(_neg_424); - float _abs_428; - asm("abs.f32 %0, %1;" : "=f"(_abs_428) : "f"(rp_z_7.y)); - float _neg_427; - asm("neg.f32 %0, %1;" : "=f"(_neg_427) : "f"(_abs_428)); - float _exp_426 = __nv_expf(_neg_427); - float2 rp_e_7 = make_float2(_exp_423, _exp_426); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_426), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_429; + asm("abs.f32 %0, %1;" : "=f"(_abs_429) : "f"(rp_z_7.x)); + float _neg_428; + asm("neg.f32 %0, %1;" : "=f"(_neg_428) : "f"(_abs_429)); + float _exp_427 = __nv_expf(_neg_428); + float _abs_432; + asm("abs.f32 %0, %1;" : "=f"(_abs_432) : "f"(rp_z_7.y)); + float _neg_431; + asm("neg.f32 %0, %1;" : "=f"(_neg_431) : "f"(_abs_432)); + float _exp_430 = __nv_expf(_neg_431); + float2 rp_e_7 = make_float2(_exp_427, _exp_430); float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_429 = approx_rcp(rp_den_7.x); - float _rcp_430 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_431 = make_float2(_rcp_429, _rcp_430); + float _rcp_433 = approx_rcp(rp_den_7.x); + float _rcp_434 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_435 = make_float2(_rcp_433, _rcp_434); float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_431)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_435)); float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_432 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_436 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_432)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_436)); u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_433 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_437 = make_float2(gacc[4], gacc[4]); float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_433), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_436; - asm("abs.f32 %0, %1;" : "=f"(_abs_436) : "f"(rp_z_8.x)); - float _neg_435; - asm("neg.f32 %0, %1;" : "=f"(_neg_435) : "f"(_abs_436)); - float _exp_434 = __nv_expf(_neg_435); - float _abs_439; - asm("abs.f32 %0, %1;" : "=f"(_abs_439) : "f"(rp_z_8.y)); - float _neg_438; - asm("neg.f32 %0, %1;" : "=f"(_neg_438) : "f"(_abs_439)); - float _exp_437 = __nv_expf(_neg_438); - float2 rp_e_8 = make_float2(_exp_434, _exp_437); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_437), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_440; + asm("abs.f32 %0, %1;" : "=f"(_abs_440) : "f"(rp_z_8.x)); + float _neg_439; + asm("neg.f32 %0, %1;" : "=f"(_neg_439) : "f"(_abs_440)); + float _exp_438 = __nv_expf(_neg_439); + float _abs_443; + asm("abs.f32 %0, %1;" : "=f"(_abs_443) : "f"(rp_z_8.y)); + float _neg_442; + asm("neg.f32 %0, %1;" : "=f"(_neg_442) : "f"(_abs_443)); + float _exp_441 = __nv_expf(_neg_442); + float2 rp_e_8 = make_float2(_exp_438, _exp_441); float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_440 = approx_rcp(rp_den_8.x); - float _rcp_441 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_442 = make_float2(_rcp_440, _rcp_441); + float _rcp_444 = approx_rcp(rp_den_8.x); + float _rcp_445 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_446 = make_float2(_rcp_444, _rcp_445); float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_442)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_446)); float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_443 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_447 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_443)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_447)); u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_444 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_448 = make_float2(gacc[4], gacc[4]); float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_444), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_447; - asm("abs.f32 %0, %1;" : "=f"(_abs_447) : "f"(rp_z_9.x)); - float _neg_446; - asm("neg.f32 %0, %1;" : "=f"(_neg_446) : "f"(_abs_447)); - float _exp_445 = __nv_expf(_neg_446); - float _abs_450; - asm("abs.f32 %0, %1;" : "=f"(_abs_450) : "f"(rp_z_9.y)); - float _neg_449; - asm("neg.f32 %0, %1;" : "=f"(_neg_449) : "f"(_abs_450)); - float _exp_448 = __nv_expf(_neg_449); - float2 rp_e_9 = make_float2(_exp_445, _exp_448); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_448), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_451; + asm("abs.f32 %0, %1;" : "=f"(_abs_451) : "f"(rp_z_9.x)); + float _neg_450; + asm("neg.f32 %0, %1;" : "=f"(_neg_450) : "f"(_abs_451)); + float _exp_449 = __nv_expf(_neg_450); + float _abs_454; + asm("abs.f32 %0, %1;" : "=f"(_abs_454) : "f"(rp_z_9.y)); + float _neg_453; + asm("neg.f32 %0, %1;" : "=f"(_neg_453) : "f"(_abs_454)); + float _exp_452 = __nv_expf(_neg_453); + float2 rp_e_9 = make_float2(_exp_449, _exp_452); float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_451 = approx_rcp(rp_den_9.x); - float _rcp_452 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_453 = make_float2(_rcp_451, _rcp_452); + float _rcp_455 = approx_rcp(rp_den_9.x); + float _rcp_456 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_457 = make_float2(_rcp_455, _rcp_456); float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_453)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_457)); float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_454 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_458 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_454)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_455 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_459 = make_float2(gacc[4], gacc[4]); float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_455), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_458; - asm("abs.f32 %0, %1;" : "=f"(_abs_458) : "f"(rp_z_10.x)); - float _neg_457; - asm("neg.f32 %0, %1;" : "=f"(_neg_457) : "f"(_abs_458)); - float _exp_456 = __nv_expf(_neg_457); - float _abs_461; - asm("abs.f32 %0, %1;" : "=f"(_abs_461) : "f"(rp_z_10.y)); - float _neg_460; - asm("neg.f32 %0, %1;" : "=f"(_neg_460) : "f"(_abs_461)); - float _exp_459 = __nv_expf(_neg_460); - float2 rp_e_10 = make_float2(_exp_456, _exp_459); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_459), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_462; + asm("abs.f32 %0, %1;" : "=f"(_abs_462) : "f"(rp_z_10.x)); + float _neg_461; + asm("neg.f32 %0, %1;" : "=f"(_neg_461) : "f"(_abs_462)); + float _exp_460 = __nv_expf(_neg_461); + float _abs_465; + asm("abs.f32 %0, %1;" : "=f"(_abs_465) : "f"(rp_z_10.y)); + float _neg_464; + asm("neg.f32 %0, %1;" : "=f"(_neg_464) : "f"(_abs_465)); + float _exp_463 = __nv_expf(_neg_464); + float2 rp_e_10 = make_float2(_exp_460, _exp_463); float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_462 = approx_rcp(rp_den_10.x); - float _rcp_463 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_464 = make_float2(_rcp_462, _rcp_463); + float _rcp_466 = approx_rcp(rp_den_10.x); + float _rcp_467 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_468 = make_float2(_rcp_466, _rcp_467); float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_464)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_468)); float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_465 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_469 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_465)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -5064,54 +4689,54 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_466; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_466) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_466; + uint32_t _prmt_470; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_470; int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_467 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_467)); - *(reinterpret_cast(_gst_467) + (0)) = pq_8[0]; + auto _gst_471 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_471)); + *(reinterpret_cast(_gst_471) + (0)) = pq_8[0]; } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_468; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_468) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_468; + uint32_t _prmt_472; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_472; int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_469 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_469)); - *(reinterpret_cast(_gst_469) + (0)) = pq_9[0]; + auto _gst_473 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_473)); + *(reinterpret_cast(_gst_473) + (0)) = pq_9[0]; } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_470; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_470; + uint32_t _prmt_474; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_474) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_474; int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_471 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_471)); - *(reinterpret_cast(_gst_471) + (0)) = pq_10[0]; + auto _gst_475 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_475)); + *(reinterpret_cast(_gst_475) + (0)) = pq_10[0]; } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_472; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_472; + uint32_t _prmt_476; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_476) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_476; int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_473 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_473)); - *(reinterpret_cast(_gst_473) + (0)) = pq_11[0]; + auto _gst_477 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_477)); + *(reinterpret_cast(_gst_477) + (0)) = pq_11[0]; } } int t_a_16 = 16 + 8 * th; @@ -5176,47 +4801,47 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_474; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_474) : "f"(s_a_1), "f"(nm_t_3)); + float _sub_478; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_478) : "f"(s_a_1), "f"(nm_t_3)); float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_474), "f"(pa_1)); - float _sub_475; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_475) : "f"(pa_1), "f"(nm_t_3)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_478), "f"(pa_1)); + float _sub_479; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_479) : "f"(pa_1), "f"(nm_t_3)); float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_475), "f"(s_a_1)); - float _abs_477; - asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(s_a_1)); - float _abs_478; - asm("abs.f32 %0, %1;" : "=f"(_abs_478) : "f"(pa_1)); - uint32_t _setp_476_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_476_bits) : "f"(_abs_477), "f"(_abs_478)); - bool _setp_476 = _setp_476_bits != 0; - float nm_err_3 = ((_setp_476) ? nm_es_3 : nm_ex_3); - float _add_479; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_479) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_479; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_479), "f"(s_a_1)); + float _abs_481; + asm("abs.f32 %0, %1;" : "=f"(_abs_481) : "f"(s_a_1)); + float _abs_482; + asm("abs.f32 %0, %1;" : "=f"(_abs_482) : "f"(pa_1)); + uint32_t _setp_480_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_480_bits) : "f"(_abs_481), "f"(_abs_482)); + bool _setp_480 = _setp_480_bits != 0; + float nm_err_3 = ((_setp_480) ? nm_es_3 : nm_ex_3); + float _add_483; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_483) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_483; s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_480; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_480) : "f"(s_b_1), "f"(nm_t_4)); + float _sub_484; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_484) : "f"(s_b_1), "f"(nm_t_4)); float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_480), "f"(pb_1)); - float _sub_481; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_481) : "f"(pb_1), "f"(nm_t_4)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_484), "f"(pb_1)); + float _sub_485; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_485) : "f"(pb_1), "f"(nm_t_4)); float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_481), "f"(s_b_1)); - float _abs_483; - asm("abs.f32 %0, %1;" : "=f"(_abs_483) : "f"(s_b_1)); - float _abs_484; - asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(pb_1)); - uint32_t _setp_482_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_482_bits) : "f"(_abs_483), "f"(_abs_484)); - bool _setp_482 = _setp_482_bits != 0; - float nm_err_4 = ((_setp_482) ? nm_es_4 : nm_ex_4); - float _add_485; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_485) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_485; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_485), "f"(s_b_1)); + float _abs_487; + asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(s_b_1)); + float _abs_488; + asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(pb_1)); + uint32_t _setp_486_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_486_bits) : "f"(_abs_487), "f"(_abs_488)); + bool _setp_486 = _setp_486_bits != 0; + float nm_err_4 = ((_setp_486) ? nm_es_4 : nm_ex_4); + float _add_489; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_489) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_489; s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5224,8 +4849,8 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_486 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_486), "f"(off_acc[0]) : "memory"); + uint32_t _addr_490 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_490), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5249,9 +4874,9 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1514 = r_322[0]; + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1514 = r_280[0]; float gc_1515 = gn_1514 - 21.640425f; int eb_1516 = 98304 + th * -32768; int ew_1517 = eb_1516 >> 2; @@ -5276,348 +4901,388 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1524[1]; s_run_1524[0] = 0.0f; - int t_16 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1517 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1517 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1517 + 2944 + d_lane]; - float bt_t_16 = btv_1523[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1514); - float e_dg_16 = approx_exp2(g_t_16 - gc_1515); - float dq2_16 = _tmem_load_26[7] * e_dg_16; - float dk2_16 = _tmem_load_27[7] * e_dg_16; - dbp_1522[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_28[7]; - float dg_o_16 = __fmaf_rn(q_t_1_24, dq2_16, (dk2b_16 - _tmem_load_28[7]) * k_t_16) + dgf_16; - s_run_1524[0] = s_run_1524[0] + dg_o_16; + int t_8 = 8 * th + 7; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_8 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_8 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_8 = __uint_as_float(r_283[0] << 16); + float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); + float dqf_8 = pool_f32[ew_1517 + 896 + d_lane]; + float dkf_8 = pool_f32[ew_1517 + 1920 + d_lane]; + float dgf_8 = pool_f32[ew_1517 + 2944 + d_lane]; + float bt_t_8 = btv_1523[7]; + float e_off_8 = approx_exp2(g_t_8 - gn_1514); + float e_dg_8 = approx_exp2(g_t_8 - gc_1515); + float dq2_8 = _tmem_load_26[7] * e_dg_8; + float dk2_8 = _tmem_load_27[7] * e_dg_8; + dbp_1522[7] = dk2_8 * k_t_8; + float dk2b_8 = dk2_8 * bt_t_8; + float dq_o_8 = dq2_8 + dqf_8; + float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_28[7]; + float qe_7 = q_t_1_8 * e_dg_8; + float kp_19 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_8) * _tmem_load_27[7]); + float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_28[7], k_t_8, cr_7)) + dgf_8; + s_run_1524[0] = s_run_1524[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_16 = t_16 * qrs; - if (t_16 < clen) { + int qoff_8 = t_8 * qrs; + if (t_8 < clen) { { - auto _gst_487 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_487)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_487) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_491 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_491)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_488 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_488)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_488) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_492 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_492)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_8; } } - int t_17 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1517 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1517 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1517 + 2816 + d_lane]; - float bt_t_17 = btv_1523[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1514); - float e_dg_17 = approx_exp2(g_t_17 - gc_1515); - float dq2_17 = _tmem_load_26[6] * e_dg_17; - float dk2_17 = _tmem_load_27[6] * e_dg_17; - dbp_1522[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[6]; - float dg_o_17 = __fmaf_rn(q_t_1_25, dq2_17, (dk2b_17 - _tmem_load_28[6]) * k_t_17) + dgf_17; - s_run_1524[0] = s_run_1524[0] + dg_o_17; + int t_9 = 8 * th + 6; + float r_284[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_9 = r_284[0]; + unsigned int r_285[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_9 = __uint_as_float(r_285[0] << 16); + unsigned int r_286[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_9 = __uint_as_float(r_286[0] << 16); + float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); + float dqf_9 = pool_f32[ew_1517 + 768 + d_lane]; + float dkf_9 = pool_f32[ew_1517 + 1792 + d_lane]; + float dgf_9 = pool_f32[ew_1517 + 2816 + d_lane]; + float bt_t_9 = btv_1523[6]; + float e_off_9 = approx_exp2(g_t_9 - gn_1514); + float e_dg_9 = approx_exp2(g_t_9 - gc_1515); + float dq2_9 = _tmem_load_26[6] * e_dg_9; + float dk2_9 = _tmem_load_27[6] * e_dg_9; + dbp_1522[6] = dk2_9 * k_t_9; + float dk2b_9 = dk2_9 * bt_t_9; + float dq_o_9 = dq2_9 + dqf_9; + float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_28[6]; + float qe_8 = q_t_1_9 * e_dg_9; + float kp_20 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_9) * _tmem_load_27[6]); + float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_28[6], k_t_9, cr_8)) + dgf_9; + s_run_1524[0] = s_run_1524[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_17 = t_17 * qrs; - if (t_17 < clen) { + int qoff_9 = t_9 * qrs; + if (t_9 < clen) { { - auto _gst_489 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_489)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_489) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_493 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_493)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_490 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_490)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_490) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_494 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_494)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_9; } } - int t_18 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1517 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1517 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1517 + 2688 + d_lane]; - float bt_t_18 = btv_1523[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1514); - float e_dg_18 = approx_exp2(g_t_18 - gc_1515); - float dq2_18 = _tmem_load_26[5] * e_dg_18; - float dk2_18 = _tmem_load_27[5] * e_dg_18; - dbp_1522[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[5]; - float dg_o_18 = __fmaf_rn(q_t_1_26, dq2_18, (dk2b_18 - _tmem_load_28[5]) * k_t_18) + dgf_18; - s_run_1524[0] = s_run_1524[0] + dg_o_18; + int t_10 = 8 * th + 5; + float r_287[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_10 = r_287[0]; + unsigned int r_288[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_10 = __uint_as_float(r_288[0] << 16); + unsigned int r_289[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_10 = __uint_as_float(r_289[0] << 16); + float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); + float dqf_10 = pool_f32[ew_1517 + 640 + d_lane]; + float dkf_10 = pool_f32[ew_1517 + 1664 + d_lane]; + float dgf_10 = pool_f32[ew_1517 + 2688 + d_lane]; + float bt_t_10 = btv_1523[5]; + float e_off_10 = approx_exp2(g_t_10 - gn_1514); + float e_dg_10 = approx_exp2(g_t_10 - gc_1515); + float dq2_10 = _tmem_load_26[5] * e_dg_10; + float dk2_10 = _tmem_load_27[5] * e_dg_10; + dbp_1522[5] = dk2_10 * k_t_10; + float dk2b_10 = dk2_10 * bt_t_10; + float dq_o_10 = dq2_10 + dqf_10; + float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_28[5]; + float qe_9 = q_t_1_10 * e_dg_10; + float kp_21 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_10) * _tmem_load_27[5]); + float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_28[5], k_t_10, cr_9)) + dgf_10; + s_run_1524[0] = s_run_1524[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_18 = t_18 * qrs; - if (t_18 < clen) { + int qoff_10 = t_10 * qrs; + if (t_10 < clen) { { - auto _gst_491 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_491)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_495 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_495)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_492 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_492)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_496 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_496)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_10; } } - int t_19 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1517 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1517 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1517 + 2560 + d_lane]; - float bt_t_19 = btv_1523[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1514); - float e_dg_19 = approx_exp2(g_t_19 - gc_1515); - float dq2_19 = _tmem_load_26[4] * e_dg_19; - float dk2_19 = _tmem_load_27[4] * e_dg_19; - dbp_1522[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[4]; - float dg_o_19 = __fmaf_rn(q_t_1_27, dq2_19, (dk2b_19 - _tmem_load_28[4]) * k_t_19) + dgf_19; - s_run_1524[0] = s_run_1524[0] + dg_o_19; + int t_11 = 8 * th + 4; + float r_290[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_11 = r_290[0]; + unsigned int r_291[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_11 = __uint_as_float(r_291[0] << 16); + unsigned int r_292[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_11 = __uint_as_float(r_292[0] << 16); + float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); + float dqf_11 = pool_f32[ew_1517 + 512 + d_lane]; + float dkf_11 = pool_f32[ew_1517 + 1536 + d_lane]; + float dgf_11 = pool_f32[ew_1517 + 2560 + d_lane]; + float bt_t_11 = btv_1523[4]; + float e_off_11 = approx_exp2(g_t_11 - gn_1514); + float e_dg_11 = approx_exp2(g_t_11 - gc_1515); + float dq2_11 = _tmem_load_26[4] * e_dg_11; + float dk2_11 = _tmem_load_27[4] * e_dg_11; + dbp_1522[4] = dk2_11 * k_t_11; + float dk2b_11 = dk2_11 * bt_t_11; + float dq_o_11 = dq2_11 + dqf_11; + float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_28[4]; + float qe_10 = q_t_1_11 * e_dg_11; + float kp_22 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_11) * _tmem_load_27[4]); + float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_28[4], k_t_11, cr_10)) + dgf_11; + s_run_1524[0] = s_run_1524[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_19 = t_19 * qrs; - if (t_19 < clen) { + int qoff_11 = t_11 * qrs; + if (t_11 < clen) { { - auto _gst_493 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_493)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_497 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_497)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_494 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_494)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_498 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_498)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_11; } } - int t_20 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1517 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1517 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1517 + 2432 + d_lane]; - float bt_t_20 = btv_1523[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1514); - float e_dg_20 = approx_exp2(g_t_20 - gc_1515); - float dq2_20 = _tmem_load_26[3] * e_dg_20; - float dk2_20 = _tmem_load_27[3] * e_dg_20; - dbp_1522[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[3]; - float dg_o_20 = __fmaf_rn(q_t_1_28, dq2_20, (dk2b_20 - _tmem_load_28[3]) * k_t_20) + dgf_20; - s_run_1524[0] = s_run_1524[0] + dg_o_20; + int t_12 = 8 * th + 3; + float r_293[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_12 = r_293[0]; + unsigned int r_294[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_12 = __uint_as_float(r_294[0] << 16); + unsigned int r_295[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_12 = __uint_as_float(r_295[0] << 16); + float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); + float dqf_12 = pool_f32[ew_1517 + 384 + d_lane]; + float dkf_12 = pool_f32[ew_1517 + 1408 + d_lane]; + float dgf_12 = pool_f32[ew_1517 + 2432 + d_lane]; + float bt_t_12 = btv_1523[3]; + float e_off_12 = approx_exp2(g_t_12 - gn_1514); + float e_dg_12 = approx_exp2(g_t_12 - gc_1515); + float dq2_12 = _tmem_load_26[3] * e_dg_12; + float dk2_12 = _tmem_load_27[3] * e_dg_12; + dbp_1522[3] = dk2_12 * k_t_12; + float dk2b_12 = dk2_12 * bt_t_12; + float dq_o_12 = dq2_12 + dqf_12; + float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_28[3]; + float qe_11 = q_t_1_12 * e_dg_12; + float kp_23 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_12) * _tmem_load_27[3]); + float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_28[3], k_t_12, cr_11)) + dgf_12; + s_run_1524[0] = s_run_1524[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_20 = t_20 * qrs; - if (t_20 < clen) { + int qoff_12 = t_12 * qrs; + if (t_12 < clen) { { - auto _gst_495 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_495)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_499 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_499)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_496 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_496)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_500 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_500)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_12; } } - int t_21 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1517 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1517 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1517 + 2304 + d_lane]; - float bt_t_21 = btv_1523[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1514); - float e_dg_21 = approx_exp2(g_t_21 - gc_1515); - float dq2_21 = _tmem_load_26[2] * e_dg_21; - float dk2_21 = _tmem_load_27[2] * e_dg_21; - dbp_1522[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[2]; - float dg_o_21 = __fmaf_rn(q_t_1_29, dq2_21, (dk2b_21 - _tmem_load_28[2]) * k_t_21) + dgf_21; - s_run_1524[0] = s_run_1524[0] + dg_o_21; + int t_13 = 8 * th + 2; + float r_296[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_13 = r_296[0]; + unsigned int r_297[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_13 = __uint_as_float(r_297[0] << 16); + unsigned int r_298[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_13 = __uint_as_float(r_298[0] << 16); + float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); + float dqf_13 = pool_f32[ew_1517 + 256 + d_lane]; + float dkf_13 = pool_f32[ew_1517 + 1280 + d_lane]; + float dgf_13 = pool_f32[ew_1517 + 2304 + d_lane]; + float bt_t_13 = btv_1523[2]; + float e_off_13 = approx_exp2(g_t_13 - gn_1514); + float e_dg_13 = approx_exp2(g_t_13 - gc_1515); + float dq2_13 = _tmem_load_26[2] * e_dg_13; + float dk2_13 = _tmem_load_27[2] * e_dg_13; + dbp_1522[2] = dk2_13 * k_t_13; + float dk2b_13 = dk2_13 * bt_t_13; + float dq_o_13 = dq2_13 + dqf_13; + float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_28[2]; + float qe_12 = q_t_1_13 * e_dg_13; + float kp_24 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_13) * _tmem_load_27[2]); + float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_28[2], k_t_13, cr_12)) + dgf_13; + s_run_1524[0] = s_run_1524[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_21 = t_21 * qrs; - if (t_21 < clen) { + int qoff_13 = t_13 * qrs; + if (t_13 < clen) { { - auto _gst_497 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_497)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_501 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_501)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_498 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_498)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_502 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_502)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_13; } } - int t_22 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1517 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1517 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1517 + 2176 + d_lane]; - float bt_t_22 = btv_1523[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1514); - float e_dg_22 = approx_exp2(g_t_22 - gc_1515); - float dq2_22 = _tmem_load_26[1] * e_dg_22; - float dk2_22 = _tmem_load_27[1] * e_dg_22; - dbp_1522[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[1]; - float dg_o_22 = __fmaf_rn(q_t_1_30, dq2_22, (dk2b_22 - _tmem_load_28[1]) * k_t_22) + dgf_22; - s_run_1524[0] = s_run_1524[0] + dg_o_22; + int t_14 = 8 * th + 1; + float r_299[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_14 = r_299[0]; + unsigned int r_300[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_14 = __uint_as_float(r_300[0] << 16); + unsigned int r_301[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_14 = __uint_as_float(r_301[0] << 16); + float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); + float dqf_14 = pool_f32[ew_1517 + 128 + d_lane]; + float dkf_14 = pool_f32[ew_1517 + 1152 + d_lane]; + float dgf_14 = pool_f32[ew_1517 + 2176 + d_lane]; + float bt_t_14 = btv_1523[1]; + float e_off_14 = approx_exp2(g_t_14 - gn_1514); + float e_dg_14 = approx_exp2(g_t_14 - gc_1515); + float dq2_14 = _tmem_load_26[1] * e_dg_14; + float dk2_14 = _tmem_load_27[1] * e_dg_14; + dbp_1522[1] = dk2_14 * k_t_14; + float dk2b_14 = dk2_14 * bt_t_14; + float dq_o_14 = dq2_14 + dqf_14; + float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_28[1]; + float qe_13 = q_t_1_14 * e_dg_14; + float kp_25 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_14) * _tmem_load_27[1]); + float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_28[1], k_t_14, cr_13)) + dgf_14; + s_run_1524[0] = s_run_1524[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_22 = t_22 * qrs; - if (t_22 < clen) { + int qoff_14 = t_14 * qrs; + if (t_14 < clen) { { - auto _gst_499 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_499)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_503 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_503)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_503) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_500 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_500)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_504 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_504)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_504) + (0)) = (__nv_bfloat16)dk_o_14; } } - int t_23 = 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1517 + d_lane]; - float dkf_23 = pool_f32[ew_1517 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1517 + 2048 + d_lane]; - float bt_t_23 = btv_1523[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1514); - float e_dg_23 = approx_exp2(g_t_23 - gc_1515); - float dq2_23 = _tmem_load_26[0] * e_dg_23; - float dk2_23 = _tmem_load_27[0] * e_dg_23; - dbp_1522[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[0]; - float dg_o_23 = __fmaf_rn(q_t_1_31, dq2_23, (dk2b_23 - _tmem_load_28[0]) * k_t_23) + dgf_23; - s_run_1524[0] = s_run_1524[0] + dg_o_23; + int t_15 = 8 * th; + float r_302[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_15 = r_302[0]; + unsigned int r_303[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_15 = __uint_as_float(r_303[0] << 16); + unsigned int r_304[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_15 = __uint_as_float(r_304[0] << 16); + float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); + float dqf_15 = pool_f32[ew_1517 + d_lane]; + float dkf_15 = pool_f32[ew_1517 + 1024 + d_lane]; + float dgf_15 = pool_f32[ew_1517 + 2048 + d_lane]; + float bt_t_15 = btv_1523[0]; + float e_off_15 = approx_exp2(g_t_15 - gn_1514); + float e_dg_15 = approx_exp2(g_t_15 - gc_1515); + float dq2_15 = _tmem_load_26[0] * e_dg_15; + float dk2_15 = _tmem_load_27[0] * e_dg_15; + dbp_1522[0] = dk2_15 * k_t_15; + float dk2b_15 = dk2_15 * bt_t_15; + float dq_o_15 = dq2_15 + dqf_15; + float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_28[0]; + float qe_14 = q_t_1_15 * e_dg_15; + float kp_26 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_15) * _tmem_load_27[0]); + float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_28[0], k_t_15, cr_14)) + dgf_15; + s_run_1524[0] = s_run_1524[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_23 = t_23 * qrs; - if (t_23 < clen) { + int qoff_15 = t_15 * qrs; + if (t_15 < clen) { { - auto _gst_501 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_501)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_505 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_505)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_505) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_502 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_502)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_506 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_506)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_506) + (0)) = (__nv_bfloat16)dk_o_15; } } float tot_1693 = s_run_1524[0]; @@ -5625,38 +5290,38 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d int b3_1695 = lane_1 >> 3 & 1; int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; - float send_12 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_503 = __shfl_xor_sync(0xFFFFFFFF, send_12, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_503; - float send_13 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_504; - float send_14 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_505 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_505; - float send_15 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_506; + float send_14 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); + float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_507; + float send_15 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); + float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_508; + float send_16 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); + float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_509; + float send_17 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); + float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, send_17, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_510; float v2_1702[2]; - float send_16 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_16, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_507; - float send_17 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_508; + float send_18 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); + float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_511; + float send_19 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); + float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, send_19, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_512; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_509; - float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_510; - float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_511; + float _shfl_xor_sync_513 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_513; + float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_514; + float _shfl_xor_sync_515 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_515; tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_512 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_512), "f"(tot_1693) : "memory"); + uint32_t _addr_516 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_516), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5688,132 +5353,132 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_513 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_517 = make_float2(gacc[4], gacc[4]); float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_513), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_516; - asm("abs.f32 %0, %1;" : "=f"(_abs_516) : "f"(rp_z_11.x)); - float _neg_515; - asm("neg.f32 %0, %1;" : "=f"(_neg_515) : "f"(_abs_516)); - float _exp_514 = __nv_expf(_neg_515); - float _abs_519; - asm("abs.f32 %0, %1;" : "=f"(_abs_519) : "f"(rp_z_11.y)); - float _neg_518; - asm("neg.f32 %0, %1;" : "=f"(_neg_518) : "f"(_abs_519)); - float _exp_517 = __nv_expf(_neg_518); - float2 rp_e_11 = make_float2(_exp_514, _exp_517); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_517), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_520; + asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_11.x)); + float _neg_519; + asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); + float _exp_518 = __nv_expf(_neg_519); + float _abs_523; + asm("abs.f32 %0, %1;" : "=f"(_abs_523) : "f"(rp_z_11.y)); + float _neg_522; + asm("neg.f32 %0, %1;" : "=f"(_neg_522) : "f"(_abs_523)); + float _exp_521 = __nv_expf(_neg_522); + float2 rp_e_11 = make_float2(_exp_518, _exp_521); float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_520 = approx_rcp(rp_den_11.x); - float _rcp_521 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_522 = make_float2(_rcp_520, _rcp_521); + float _rcp_524 = approx_rcp(rp_den_11.x); + float _rcp_525 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_526 = make_float2(_rcp_524, _rcp_525); float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_522)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_526)); float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_523 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_527 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_527)); u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_524 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_528 = make_float2(gacc[4], gacc[4]); float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_524), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_527; - asm("abs.f32 %0, %1;" : "=f"(_abs_527) : "f"(rp_z_12.x)); - float _neg_526; - asm("neg.f32 %0, %1;" : "=f"(_neg_526) : "f"(_abs_527)); - float _exp_525 = __nv_expf(_neg_526); - float _abs_530; - asm("abs.f32 %0, %1;" : "=f"(_abs_530) : "f"(rp_z_12.y)); - float _neg_529; - asm("neg.f32 %0, %1;" : "=f"(_neg_529) : "f"(_abs_530)); - float _exp_528 = __nv_expf(_neg_529); - float2 rp_e_12 = make_float2(_exp_525, _exp_528); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_528), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_531; + asm("abs.f32 %0, %1;" : "=f"(_abs_531) : "f"(rp_z_12.x)); + float _neg_530; + asm("neg.f32 %0, %1;" : "=f"(_neg_530) : "f"(_abs_531)); + float _exp_529 = __nv_expf(_neg_530); + float _abs_534; + asm("abs.f32 %0, %1;" : "=f"(_abs_534) : "f"(rp_z_12.y)); + float _neg_533; + asm("neg.f32 %0, %1;" : "=f"(_neg_533) : "f"(_abs_534)); + float _exp_532 = __nv_expf(_neg_533); + float2 rp_e_12 = make_float2(_exp_529, _exp_532); float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_531 = approx_rcp(rp_den_12.x); - float _rcp_532 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_533 = make_float2(_rcp_531, _rcp_532); + float _rcp_535 = approx_rcp(rp_den_12.x); + float _rcp_536 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_537 = make_float2(_rcp_535, _rcp_536); float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_533)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_537)); float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_534 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_538 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_534)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_538)); u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_535 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_539 = make_float2(gacc[4], gacc[4]); float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_535), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_538; - asm("abs.f32 %0, %1;" : "=f"(_abs_538) : "f"(rp_z_13.x)); - float _neg_537; - asm("neg.f32 %0, %1;" : "=f"(_neg_537) : "f"(_abs_538)); - float _exp_536 = __nv_expf(_neg_537); - float _abs_541; - asm("abs.f32 %0, %1;" : "=f"(_abs_541) : "f"(rp_z_13.y)); - float _neg_540; - asm("neg.f32 %0, %1;" : "=f"(_neg_540) : "f"(_abs_541)); - float _exp_539 = __nv_expf(_neg_540); - float2 rp_e_13 = make_float2(_exp_536, _exp_539); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_539), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_542; + asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(rp_z_13.x)); + float _neg_541; + asm("neg.f32 %0, %1;" : "=f"(_neg_541) : "f"(_abs_542)); + float _exp_540 = __nv_expf(_neg_541); + float _abs_545; + asm("abs.f32 %0, %1;" : "=f"(_abs_545) : "f"(rp_z_13.y)); + float _neg_544; + asm("neg.f32 %0, %1;" : "=f"(_neg_544) : "f"(_abs_545)); + float _exp_543 = __nv_expf(_neg_544); + float2 rp_e_13 = make_float2(_exp_540, _exp_543); float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_542 = approx_rcp(rp_den_13.x); - float _rcp_543 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_544 = make_float2(_rcp_542, _rcp_543); + float _rcp_546 = approx_rcp(rp_den_13.x); + float _rcp_547 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_548 = make_float2(_rcp_546, _rcp_547); float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_544)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_548)); float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_545 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_549 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_545)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_549)); u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_546 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_550 = make_float2(gacc[4], gacc[4]); float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_546), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_549; - asm("abs.f32 %0, %1;" : "=f"(_abs_549) : "f"(rp_z_14.x)); - float _neg_548; - asm("neg.f32 %0, %1;" : "=f"(_neg_548) : "f"(_abs_549)); - float _exp_547 = __nv_expf(_neg_548); - float _abs_552; - asm("abs.f32 %0, %1;" : "=f"(_abs_552) : "f"(rp_z_14.y)); - float _neg_551; - asm("neg.f32 %0, %1;" : "=f"(_neg_551) : "f"(_abs_552)); - float _exp_550 = __nv_expf(_neg_551); - float2 rp_e_14 = make_float2(_exp_547, _exp_550); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_550), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_553; + asm("abs.f32 %0, %1;" : "=f"(_abs_553) : "f"(rp_z_14.x)); + float _neg_552; + asm("neg.f32 %0, %1;" : "=f"(_neg_552) : "f"(_abs_553)); + float _exp_551 = __nv_expf(_neg_552); + float _abs_556; + asm("abs.f32 %0, %1;" : "=f"(_abs_556) : "f"(rp_z_14.y)); + float _neg_555; + asm("neg.f32 %0, %1;" : "=f"(_neg_555) : "f"(_abs_556)); + float _exp_554 = __nv_expf(_neg_555); + float2 rp_e_14 = make_float2(_exp_551, _exp_554); float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_553 = approx_rcp(rp_den_14.x); - float _rcp_554 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_555 = make_float2(_rcp_553, _rcp_554); + float _rcp_557 = approx_rcp(rp_den_14.x); + float _rcp_558 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_559 = make_float2(_rcp_557, _rcp_558); float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_555)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_559)); float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_556 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_560 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_556)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_560)); u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -5828,54 +5493,54 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_557; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_557) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_557; + uint32_t _prmt_561; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_561; int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_558 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_558)); - *(reinterpret_cast(_gst_558) + (0)) = pq_12[0]; + auto _gst_562 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_562)); + *(reinterpret_cast(_gst_562) + (0)) = pq_12[0]; } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_559; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_559) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_559; + uint32_t _prmt_563; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_563; int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_560 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_560)); - *(reinterpret_cast(_gst_560) + (0)) = pq_13[0]; + auto _gst_564 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_564)); + *(reinterpret_cast(_gst_564) + (0)) = pq_13[0]; } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_561; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_561; + uint32_t _prmt_565; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_565) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_565; int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_562 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_562)); - *(reinterpret_cast(_gst_562) + (0)) = pq_14[0]; + auto _gst_566 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_566)); + *(reinterpret_cast(_gst_566) + (0)) = pq_14[0]; } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_563; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_563; + uint32_t _prmt_567; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_567) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_567; int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_564 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_564)); - *(reinterpret_cast(_gst_564) + (0)) = pq_15[0]; + auto _gst_568 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_568)); + *(reinterpret_cast(_gst_568) + (0)) = pq_15[0]; } } int t_a_24 = 8 * th; @@ -5940,47 +5605,47 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_565; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_565) : "f"(s_a_2), "f"(nm_t_5)); + float _sub_569; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_569) : "f"(s_a_2), "f"(nm_t_5)); float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_565), "f"(pa_2)); - float _sub_566; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_566) : "f"(pa_2), "f"(nm_t_5)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_569), "f"(pa_2)); + float _sub_570; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_570) : "f"(pa_2), "f"(nm_t_5)); float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_566), "f"(s_a_2)); - float _abs_568; - asm("abs.f32 %0, %1;" : "=f"(_abs_568) : "f"(s_a_2)); - float _abs_569; - asm("abs.f32 %0, %1;" : "=f"(_abs_569) : "f"(pa_2)); - uint32_t _setp_567_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_567_bits) : "f"(_abs_568), "f"(_abs_569)); - bool _setp_567 = _setp_567_bits != 0; - float nm_err_5 = ((_setp_567) ? nm_es_5 : nm_ex_5); - float _add_570; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_570; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_570), "f"(s_a_2)); + float _abs_572; + asm("abs.f32 %0, %1;" : "=f"(_abs_572) : "f"(s_a_2)); + float _abs_573; + asm("abs.f32 %0, %1;" : "=f"(_abs_573) : "f"(pa_2)); + uint32_t _setp_571_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_571_bits) : "f"(_abs_572), "f"(_abs_573)); + bool _setp_571 = _setp_571_bits != 0; + float nm_err_5 = ((_setp_571) ? nm_es_5 : nm_ex_5); + float _add_574; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_574; s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_571; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(s_b_2), "f"(nm_t_6)); + float _sub_575; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_575) : "f"(s_b_2), "f"(nm_t_6)); float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_571), "f"(pb_2)); - float _sub_572; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_572) : "f"(pb_2), "f"(nm_t_6)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_575), "f"(pb_2)); + float _sub_576; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_576) : "f"(pb_2), "f"(nm_t_6)); float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_572), "f"(s_b_2)); - float _abs_574; - asm("abs.f32 %0, %1;" : "=f"(_abs_574) : "f"(s_b_2)); - float _abs_575; - asm("abs.f32 %0, %1;" : "=f"(_abs_575) : "f"(pb_2)); - uint32_t _setp_573_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_573_bits) : "f"(_abs_574), "f"(_abs_575)); - bool _setp_573 = _setp_573_bits != 0; - float nm_err_6 = ((_setp_573) ? nm_es_6 : nm_ex_6); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_576; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_576), "f"(s_b_2)); + float _abs_578; + asm("abs.f32 %0, %1;" : "=f"(_abs_578) : "f"(s_b_2)); + float _abs_579; + asm("abs.f32 %0, %1;" : "=f"(_abs_579) : "f"(pb_2)); + uint32_t _setp_577_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_577_bits) : "f"(_abs_578), "f"(_abs_579)); + bool _setp_577 = _setp_577_bits != 0; + float nm_err_6 = ((_setp_577) ? nm_es_6 : nm_ex_6); + float _add_580; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_580; s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -6019,60 +5684,60 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_578; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_578) : "f"(ts_s), "f"(ts_bb)); - float _sub_577; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_577) : "f"(s_a_3), "f"(_sub_578)); - float _sub_579; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_579) : "f"(other[0]), "f"(ts_bb)); + float _sub_582; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s), "f"(ts_bb)); + float _sub_581; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(s_a_3), "f"(_sub_582)); + float _sub_583; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(other[0]), "f"(ts_bb)); float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_577), "f"(_sub_579)); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_a_3), "f"(other[1])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_581), "f"(_sub_583)); + float _add_584; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(c_a_3), "f"(other[1])); float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_580), "f"(ts_e)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_584), "f"(ts_e)); float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_582; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_581; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(ts_s), "f"(_sub_582)); - float _sub_583; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(pm_c), "f"(ts_bb_0)); + float _sub_586; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_585; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s), "f"(_sub_586)); + float _sub_587; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_587) : "f"(pm_c), "f"(ts_bb_0)); float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_581), "f"(_sub_583)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_585), "f"(_sub_587)); s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_585; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_584; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_584) : "f"(s_b_3), "f"(_sub_585)); - float _sub_586; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(other[2]), "f"(ts_bb_1)); + float _sub_589; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_588; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(s_b_3), "f"(_sub_589)); + float _sub_590; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(other[2]), "f"(ts_bb_1)); float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_584), "f"(_sub_586)); - float _add_587; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_587) : "f"(c_b_3), "f"(other[3])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_588), "f"(_sub_590)); + float _add_591; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_591) : "f"(c_b_3), "f"(other[3])); float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_587), "f"(ts_e_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_591), "f"(ts_e_1)); float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_589; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_588; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(ts_s_1), "f"(_sub_589)); - float _sub_590; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(pm_c_0), "f"(ts_bb_2)); + float _sub_593; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_593) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_592; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_592) : "f"(ts_s_1), "f"(_sub_593)); + float _sub_594; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_594) : "f"(pm_c_0), "f"(ts_bb_2)); float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_588), "f"(_sub_590)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_592), "f"(_sub_594)); s_b_3 = ts_s_2; c_b_3 = ts_e_2; gacc[0] = s_a_3; @@ -6093,7 +5758,7 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_1706; pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1484; pool_f32[28160 + qd * 64 + 32 + 8 * th + e_star] = v1; - pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_1241; + pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_0; } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync 1, 256;" ::: "memory"); @@ -6183,41 +5848,41 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + for (int r_305 = 0; r_305 < 16; r_305++) { + int t_16 = r_305; + if (t_16 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6289,41 +5954,41 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_24 = 16 + r_348; - if (t_24 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + for (int r_306 = 0; r_306 < 16; r_306++) { + int t_17 = 16 + r_306; + if (t_17 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6439,41 +6104,41 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_25 = 32 + r_349; - if (t_25 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + for (int r_307 = 0; r_307 < 16; r_307++) { + int t_18 = 32 + r_307; + if (t_18 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6621,41 +6286,41 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_26 = 48 + r_350; - if (t_26 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + for (int r_308 = 0; r_308 < 16; r_308++) { + int t_19 = 48 + r_308; + if (t_19 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu index 26cc6d1aab8..51e16f3e7b4 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2246,15 +2269,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2275,15 +2304,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2304,15 +2339,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2333,15 +2374,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2502,15 +2573,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2531,15 +2608,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2560,15 +2643,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2589,15 +2678,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2636,15 +2731,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2665,15 +2766,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2694,15 +2801,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2723,15 +2836,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2802,37 +2921,37 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2953,9 +3072,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_0 = __uint_as_float(r_250[0] << 16); float q_t0_0 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew + d_lane]; - float dkf_0 = pool_f32[ew + 1024 + d_lane]; - float dgf_0 = pool_f32[ew + 2048 + d_lane]; float bt_t_0 = btv[0]; float e_off_0 = approx_exp2(g_t_0 - gn); float e_dg_0 = approx_exp2(g_t_0 - gc_3); @@ -2963,22 +3079,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_0 = __fmaf_rn(ko[0], e_off_0, kdg[0] * e_dg_0); dbp[0] = dk2_0 * k_t_0; float dk2b_0 = dk2_0 * bt_t_0; + float q_r_0 = q_t0_0; + float qe = q_r_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dqf_0 = pool_f32[ew + d_lane]; + float dkf_0 = pool_f32[ew + 1024 + d_lane]; + float dgf_0 = pool_f32[ew + 2048 + d_lane]; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; - float q_r_0 = q_t0_0; - float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0)) + dgf_0; + float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr))) + dgf_0; dgo[0] = dg_o_0; int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_248 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_249 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -2992,9 +3116,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_1 = __uint_as_float(r_253[0] << 16); float q_t0_1 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew + 128 + d_lane]; - float dkf_1 = pool_f32[ew + 1152 + d_lane]; - float dgf_1 = pool_f32[ew + 2176 + d_lane]; float bt_t_1 = btv[1]; float e_off_1 = approx_exp2(g_t_1 - gn); float e_dg_1 = approx_exp2(g_t_1 - gc_3); @@ -3002,22 +3123,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_1 = __fmaf_rn(ko[1], e_off_1, kdg[1] * e_dg_1); dbp[1] = dk2_1 * k_t_1; float dk2b_1 = dk2_1 * bt_t_1; + float q_r_1 = q_t0_1; + float qe_0 = q_r_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dqf_1 = pool_f32[ew + 128 + d_lane]; + float dkf_1 = pool_f32[ew + 1152 + d_lane]; + float dgf_1 = pool_f32[ew + 2176 + d_lane]; float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; - float q_r_1 = q_t0_1; - float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1)) + dgf_1; + float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0))) + dgf_1; dgo[1] = dg_o_1; int qoff_1 = t_1 * qrs; if (t_1 < clen) { { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_250 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_251 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3031,9 +3160,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_2 = __uint_as_float(r_256[0] << 16); float q_t0_2 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew + 256 + d_lane]; - float dkf_2 = pool_f32[ew + 1280 + d_lane]; - float dgf_2 = pool_f32[ew + 2304 + d_lane]; float bt_t_2 = btv[2]; float e_off_2 = approx_exp2(g_t_2 - gn); float e_dg_2 = approx_exp2(g_t_2 - gc_3); @@ -3041,22 +3167,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_2 = __fmaf_rn(ko[2], e_off_2, kdg[2] * e_dg_2); dbp[2] = dk2_2 * k_t_2; float dk2b_2 = dk2_2 * bt_t_2; + float q_r_2 = q_t0_2; + float qe_1 = q_r_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dqf_2 = pool_f32[ew + 256 + d_lane]; + float dkf_2 = pool_f32[ew + 1280 + d_lane]; + float dgf_2 = pool_f32[ew + 2304 + d_lane]; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; - float q_r_2 = q_t0_2; - float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2)) + dgf_2; + float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1))) + dgf_2; dgo[2] = dg_o_2; int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_252 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_253 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3070,9 +3204,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_3 = __uint_as_float(r_259[0] << 16); float q_t0_3 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew + 384 + d_lane]; - float dkf_3 = pool_f32[ew + 1408 + d_lane]; - float dgf_3 = pool_f32[ew + 2432 + d_lane]; float bt_t_3 = btv[3]; float e_off_3 = approx_exp2(g_t_3 - gn); float e_dg_3 = approx_exp2(g_t_3 - gc_3); @@ -3080,22 +3211,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_3 = __fmaf_rn(ko[3], e_off_3, kdg[3] * e_dg_3); dbp[3] = dk2_3 * k_t_3; float dk2b_3 = dk2_3 * bt_t_3; + float q_r_3 = q_t0_3; + float qe_2 = q_r_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dqf_3 = pool_f32[ew + 384 + d_lane]; + float dkf_3 = pool_f32[ew + 1408 + d_lane]; + float dgf_3 = pool_f32[ew + 2432 + d_lane]; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; - float q_r_3 = q_t0_3; - float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3)) + dgf_3; + float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2))) + dgf_3; dgo[3] = dg_o_3; int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_254 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_255 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3109,9 +3248,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_4 = __uint_as_float(r_262[0] << 16); float q_t0_4 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew + 512 + d_lane]; - float dkf_4 = pool_f32[ew + 1536 + d_lane]; - float dgf_4 = pool_f32[ew + 2560 + d_lane]; float bt_t_4 = btv[4]; float e_off_4 = approx_exp2(g_t_4 - gn); float e_dg_4 = approx_exp2(g_t_4 - gc_3); @@ -3119,22 +3255,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_4 = __fmaf_rn(ko[4], e_off_4, kdg[4] * e_dg_4); dbp[4] = dk2_4 * k_t_4; float dk2b_4 = dk2_4 * bt_t_4; + float q_r_4 = q_t0_4; + float qe_3 = q_r_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dqf_4 = pool_f32[ew + 512 + d_lane]; + float dkf_4 = pool_f32[ew + 1536 + d_lane]; + float dgf_4 = pool_f32[ew + 2560 + d_lane]; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; - float q_r_4 = q_t0_4; - float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4)) + dgf_4; + float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3))) + dgf_4; dgo[4] = dg_o_4; int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_256 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_257 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3148,9 +3292,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_5 = __uint_as_float(r_265[0] << 16); float q_t0_5 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew + 640 + d_lane]; - float dkf_5 = pool_f32[ew + 1664 + d_lane]; - float dgf_5 = pool_f32[ew + 2688 + d_lane]; float bt_t_5 = btv[5]; float e_off_5 = approx_exp2(g_t_5 - gn); float e_dg_5 = approx_exp2(g_t_5 - gc_3); @@ -3158,22 +3299,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_5 = __fmaf_rn(ko[5], e_off_5, kdg[5] * e_dg_5); dbp[5] = dk2_5 * k_t_5; float dk2b_5 = dk2_5 * bt_t_5; + float q_r_5 = q_t0_5; + float qe_4 = q_r_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dqf_5 = pool_f32[ew + 640 + d_lane]; + float dkf_5 = pool_f32[ew + 1664 + d_lane]; + float dgf_5 = pool_f32[ew + 2688 + d_lane]; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; - float q_r_5 = q_t0_5; - float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5)) + dgf_5; + float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4))) + dgf_5; dgo[5] = dg_o_5; int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_258 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_259 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3187,9 +3336,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_6 = __uint_as_float(r_268[0] << 16); float q_t0_6 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew + 768 + d_lane]; - float dkf_6 = pool_f32[ew + 1792 + d_lane]; - float dgf_6 = pool_f32[ew + 2816 + d_lane]; float bt_t_6 = btv[6]; float e_off_6 = approx_exp2(g_t_6 - gn); float e_dg_6 = approx_exp2(g_t_6 - gc_3); @@ -3197,22 +3343,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_6 = __fmaf_rn(ko[6], e_off_6, kdg[6] * e_dg_6); dbp[6] = dk2_6 * k_t_6; float dk2b_6 = dk2_6 * bt_t_6; + float q_r_6 = q_t0_6; + float qe_5 = q_r_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dqf_6 = pool_f32[ew + 768 + d_lane]; + float dkf_6 = pool_f32[ew + 1792 + d_lane]; + float dgf_6 = pool_f32[ew + 2816 + d_lane]; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; - float q_r_6 = q_t0_6; - float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6)) + dgf_6; + float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5))) + dgf_6; dgo[6] = dg_o_6; int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_260 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_261 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3226,9 +3380,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_7 = __uint_as_float(r_271[0] << 16); float q_t0_7 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew + 896 + d_lane]; - float dkf_7 = pool_f32[ew + 1920 + d_lane]; - float dgf_7 = pool_f32[ew + 2944 + d_lane]; float bt_t_7 = btv[7]; float e_off_7 = approx_exp2(g_t_7 - gn); float e_dg_7 = approx_exp2(g_t_7 - gc_3); @@ -3236,22 +3387,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_7 = __fmaf_rn(ko[7], e_off_7, kdg[7] * e_dg_7); dbp[7] = dk2_7 * k_t_7; float dk2b_7 = dk2_7 * bt_t_7; + float q_r_7 = q_t0_7; + float qe_6 = q_r_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dqf_7 = pool_f32[ew + 896 + d_lane]; + float dkf_7 = pool_f32[ew + 1920 + d_lane]; + float dgf_7 = pool_f32[ew + 2944 + d_lane]; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; - float q_r_7 = q_t0_7; - float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7)) + dgf_7; + float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6))) + dgf_7; dgo[7] = dg_o_7; int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_262 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_263 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_7; } } float s_run[1]; @@ -3280,31 +3439,31 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3351,9 +3510,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_8 = __uint_as_float(r_275[0] << 16); float q_t0_8 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_0 + d_lane]; - float dkf_8 = pool_f32[ew_0 + 1024 + d_lane]; - float dgf_8 = pool_f32[ew_0 + 2048 + d_lane]; float bt_t_8 = btv_0[0]; float e_off_8 = approx_exp2(g_t_8 - gn_0); float e_dg_8 = approx_exp2(g_t_8 - gc_4); @@ -3361,22 +3517,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_8 = __fmaf_rn(ko_0[0], e_off_8, kdg_0[0] * e_dg_8); dbp_0[0] = dk2_8 * k_t_8; float dk2b_8 = dk2_8 * bt_t_8; + float q_r_8 = q_t0_8; + float qe_7 = q_r_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dqf_8 = pool_f32[ew_0 + d_lane]; + float dkf_8 = pool_f32[ew_0 + 1024 + d_lane]; + float dgf_8 = pool_f32[ew_0 + 2048 + d_lane]; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; - float q_r_8 = q_t0_8; - float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8)) + dgf_8; + float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7))) + dgf_8; dgo_0[0] = dg_o_8; int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_273 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_274 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3390,9 +3554,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_9 = __uint_as_float(r_278[0] << 16); float q_t0_9 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_0 + 128 + d_lane]; - float dkf_9 = pool_f32[ew_0 + 1152 + d_lane]; - float dgf_9 = pool_f32[ew_0 + 2176 + d_lane]; float bt_t_9 = btv_0[1]; float e_off_9 = approx_exp2(g_t_9 - gn_0); float e_dg_9 = approx_exp2(g_t_9 - gc_4); @@ -3400,22 +3561,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_9 = __fmaf_rn(ko_0[1], e_off_9, kdg_0[1] * e_dg_9); dbp_0[1] = dk2_9 * k_t_9; float dk2b_9 = dk2_9 * bt_t_9; + float q_r_9 = q_t0_9; + float qe_8 = q_r_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dqf_9 = pool_f32[ew_0 + 128 + d_lane]; + float dkf_9 = pool_f32[ew_0 + 1152 + d_lane]; + float dgf_9 = pool_f32[ew_0 + 2176 + d_lane]; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; - float q_r_9 = q_t0_9; - float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9)) + dgf_9; + float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8))) + dgf_9; dgo_0[1] = dg_o_9; int qoff_9 = t_9 * qrs; if (t_9 < clen) { { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_275 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_276 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3429,9 +3598,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_10 = __uint_as_float(r_281[0] << 16); float q_t0_10 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_0 + 256 + d_lane]; - float dkf_10 = pool_f32[ew_0 + 1280 + d_lane]; - float dgf_10 = pool_f32[ew_0 + 2304 + d_lane]; float bt_t_10 = btv_0[2]; float e_off_10 = approx_exp2(g_t_10 - gn_0); float e_dg_10 = approx_exp2(g_t_10 - gc_4); @@ -3439,22 +3605,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_10 = __fmaf_rn(ko_0[2], e_off_10, kdg_0[2] * e_dg_10); dbp_0[2] = dk2_10 * k_t_10; float dk2b_10 = dk2_10 * bt_t_10; + float q_r_10 = q_t0_10; + float qe_9 = q_r_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dqf_10 = pool_f32[ew_0 + 256 + d_lane]; + float dkf_10 = pool_f32[ew_0 + 1280 + d_lane]; + float dgf_10 = pool_f32[ew_0 + 2304 + d_lane]; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; - float q_r_10 = q_t0_10; - float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10)) + dgf_10; + float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9))) + dgf_10; dgo_0[2] = dg_o_10; int qoff_10 = t_10 * qrs; if (t_10 < clen) { { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_277 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_278 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3468,9 +3642,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_11 = __uint_as_float(r_284[0] << 16); float q_t0_11 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_0 + 384 + d_lane]; - float dkf_11 = pool_f32[ew_0 + 1408 + d_lane]; - float dgf_11 = pool_f32[ew_0 + 2432 + d_lane]; float bt_t_11 = btv_0[3]; float e_off_11 = approx_exp2(g_t_11 - gn_0); float e_dg_11 = approx_exp2(g_t_11 - gc_4); @@ -3478,22 +3649,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_11 = __fmaf_rn(ko_0[3], e_off_11, kdg_0[3] * e_dg_11); dbp_0[3] = dk2_11 * k_t_11; float dk2b_11 = dk2_11 * bt_t_11; + float q_r_11 = q_t0_11; + float qe_10 = q_r_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dqf_11 = pool_f32[ew_0 + 384 + d_lane]; + float dkf_11 = pool_f32[ew_0 + 1408 + d_lane]; + float dgf_11 = pool_f32[ew_0 + 2432 + d_lane]; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; - float q_r_11 = q_t0_11; - float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11)) + dgf_11; + float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10))) + dgf_11; dgo_0[3] = dg_o_11; int qoff_11 = t_11 * qrs; if (t_11 < clen) { { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_279 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_280 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3507,9 +3686,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_12 = __uint_as_float(r_287[0] << 16); float q_t0_12 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_0 + 512 + d_lane]; - float dkf_12 = pool_f32[ew_0 + 1536 + d_lane]; - float dgf_12 = pool_f32[ew_0 + 2560 + d_lane]; float bt_t_12 = btv_0[4]; float e_off_12 = approx_exp2(g_t_12 - gn_0); float e_dg_12 = approx_exp2(g_t_12 - gc_4); @@ -3517,22 +3693,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_12 = __fmaf_rn(ko_0[4], e_off_12, kdg_0[4] * e_dg_12); dbp_0[4] = dk2_12 * k_t_12; float dk2b_12 = dk2_12 * bt_t_12; + float q_r_12 = q_t0_12; + float qe_11 = q_r_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dqf_12 = pool_f32[ew_0 + 512 + d_lane]; + float dkf_12 = pool_f32[ew_0 + 1536 + d_lane]; + float dgf_12 = pool_f32[ew_0 + 2560 + d_lane]; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; - float q_r_12 = q_t0_12; - float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12)) + dgf_12; + float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11))) + dgf_12; dgo_0[4] = dg_o_12; int qoff_12 = t_12 * qrs; if (t_12 < clen) { { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_281 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_282 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3546,9 +3730,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_13 = __uint_as_float(r_290[0] << 16); float q_t0_13 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_0 + 640 + d_lane]; - float dkf_13 = pool_f32[ew_0 + 1664 + d_lane]; - float dgf_13 = pool_f32[ew_0 + 2688 + d_lane]; float bt_t_13 = btv_0[5]; float e_off_13 = approx_exp2(g_t_13 - gn_0); float e_dg_13 = approx_exp2(g_t_13 - gc_4); @@ -3556,22 +3737,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_13 = __fmaf_rn(ko_0[5], e_off_13, kdg_0[5] * e_dg_13); dbp_0[5] = dk2_13 * k_t_13; float dk2b_13 = dk2_13 * bt_t_13; + float q_r_13 = q_t0_13; + float qe_12 = q_r_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dqf_13 = pool_f32[ew_0 + 640 + d_lane]; + float dkf_13 = pool_f32[ew_0 + 1664 + d_lane]; + float dgf_13 = pool_f32[ew_0 + 2688 + d_lane]; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; - float q_r_13 = q_t0_13; - float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13)) + dgf_13; + float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12))) + dgf_13; dgo_0[5] = dg_o_13; int qoff_13 = t_13 * qrs; if (t_13 < clen) { { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_283 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_284 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3585,9 +3774,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_14 = __uint_as_float(r_293[0] << 16); float q_t0_14 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_0 + 768 + d_lane]; - float dkf_14 = pool_f32[ew_0 + 1792 + d_lane]; - float dgf_14 = pool_f32[ew_0 + 2816 + d_lane]; float bt_t_14 = btv_0[6]; float e_off_14 = approx_exp2(g_t_14 - gn_0); float e_dg_14 = approx_exp2(g_t_14 - gc_4); @@ -3595,22 +3781,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_14 = __fmaf_rn(ko_0[6], e_off_14, kdg_0[6] * e_dg_14); dbp_0[6] = dk2_14 * k_t_14; float dk2b_14 = dk2_14 * bt_t_14; + float q_r_14 = q_t0_14; + float qe_13 = q_r_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dqf_14 = pool_f32[ew_0 + 768 + d_lane]; + float dkf_14 = pool_f32[ew_0 + 1792 + d_lane]; + float dgf_14 = pool_f32[ew_0 + 2816 + d_lane]; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; - float q_r_14 = q_t0_14; - float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14)) + dgf_14; + float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13))) + dgf_14; dgo_0[6] = dg_o_14; int qoff_14 = t_14 * qrs; if (t_14 < clen) { { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_285 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_286 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3624,9 +3818,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_15 = __uint_as_float(r_296[0] << 16); float q_t0_15 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_0 + 896 + d_lane]; - float dkf_15 = pool_f32[ew_0 + 1920 + d_lane]; - float dgf_15 = pool_f32[ew_0 + 2944 + d_lane]; float bt_t_15 = btv_0[7]; float e_off_15 = approx_exp2(g_t_15 - gn_0); float e_dg_15 = approx_exp2(g_t_15 - gc_4); @@ -3634,22 +3825,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_15 = __fmaf_rn(ko_0[7], e_off_15, kdg_0[7] * e_dg_15); dbp_0[7] = dk2_15 * k_t_15; float dk2b_15 = dk2_15 * bt_t_15; + float q_r_15 = q_t0_15; + float qe_14 = q_r_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dqf_15 = pool_f32[ew_0 + 896 + d_lane]; + float dkf_15 = pool_f32[ew_0 + 1920 + d_lane]; + float dgf_15 = pool_f32[ew_0 + 2944 + d_lane]; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; - float q_r_15 = q_t0_15; - float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15)) + dgf_15; + float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14))) + dgf_15; dgo_0[7] = dg_o_15; int qoff_15 = t_15 * qrs; if (t_15 < clen) { { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_287 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_288 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_15; } } float s_run_0[1]; @@ -3678,36 +3877,36 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_289; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_290; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_291; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_292; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_293; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_294; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -3725,9 +3924,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_0 = offset + sfx[0]; if (t_16 < clen) { { - auto _gst_267 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_0; + auto _gst_299 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_0; } } int t_17 = 48 + 8 * th + 1; @@ -3735,9 +3934,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_1 = offset + sfx[1]; if (t_17 < clen) { { - auto _gst_268 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1; + auto _gst_300 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1; } } int t_18 = 48 + 8 * th + 2; @@ -3745,9 +3944,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_2 = offset + sfx[2]; if (t_18 < clen) { { - auto _gst_269 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_2; + auto _gst_301 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_2; } } int t_19 = 48 + 8 * th + 3; @@ -3755,9 +3954,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_3 = offset + sfx[3]; if (t_19 < clen) { { - auto _gst_270 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_3; + auto _gst_302 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_3; } } int t_20 = 48 + 8 * th + 4; @@ -3765,9 +3964,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_4 = offset + sfx[4]; if (t_20 < clen) { { - auto _gst_271 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_4; + auto _gst_303 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_4; } } int t_21 = 48 + 8 * th + 5; @@ -3775,9 +3974,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_5 = offset + sfx[5]; if (t_21 < clen) { { - auto _gst_272 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_5; + auto _gst_304 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_5; } } int t_22 = 48 + 8 * th + 6; @@ -3785,9 +3984,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_6 = offset + sfx[6]; if (t_22 < clen) { { - auto _gst_273 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_6; + auto _gst_305 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_6; } } int t_23 = 48 + 8 * th + 7; @@ -3795,9 +3994,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_7 = offset + sfx[7]; if (t_23 < clen) { { - auto _gst_274 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_7; + auto _gst_306 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3815,9 +4014,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_8 = offset_0 + sfx_0[0]; if (t_24 < clen) { { - auto _gst_275 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_8; + auto _gst_307 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_8; } } int t_25 = 32 + 8 * th + 1; @@ -3825,9 +4024,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_9 = offset_0 + sfx_0[1]; if (t_25 < clen) { { - auto _gst_276 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_9; + auto _gst_308 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_9; } } int t_26 = 32 + 8 * th + 2; @@ -3835,9 +4034,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_10 = offset_0 + sfx_0[2]; if (t_26 < clen) { { - auto _gst_277 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_10; + auto _gst_309 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_10; } } int t_27 = 32 + 8 * th + 3; @@ -3845,9 +4044,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_11 = offset_0 + sfx_0[3]; if (t_27 < clen) { { - auto _gst_278 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_11; + auto _gst_310 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_11; } } int t_28 = 32 + 8 * th + 4; @@ -3855,9 +4054,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_12 = offset_0 + sfx_0[4]; if (t_28 < clen) { { - auto _gst_279 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_12; + auto _gst_311 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_12; } } int t_29 = 32 + 8 * th + 5; @@ -3865,9 +4064,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_13 = offset_0 + sfx_0[5]; if (t_29 < clen) { { - auto _gst_280 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_13; + auto _gst_312 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_13; } } int t_30 = 32 + 8 * th + 6; @@ -3875,9 +4074,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_14 = offset_0 + sfx_0[6]; if (t_30 < clen) { { - auto _gst_281 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_14; + auto _gst_313 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_14; } } int t_31 = 32 + 8 * th + 7; @@ -3885,15 +4084,15 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_15 = offset_0 + sfx_0[7]; if (t_31 < clen) { { - auto _gst_282 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_15; + auto _gst_314 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3940,9 +4139,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_16 = __uint_as_float(r_300[0] << 16); float q_t0_16 = ((t_32 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1 + d_lane]; - float dkf_16 = pool_f32[ew_1 + 1024 + d_lane]; - float dgf_16 = pool_f32[ew_1 + 2048 + d_lane]; float bt_t_16 = btv_1[0]; float e_off_16 = approx_exp2(g_t_16 - gn_1); float e_dg_16 = approx_exp2(g_t_16 - gc_5); @@ -3950,22 +4146,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_16 = __fmaf_rn(ko_1[0], e_off_16, kdg_1[0] * e_dg_16); dbp_1[0] = dk2_16 * k_t_16; float dk2b_16 = dk2_16 * bt_t_16; + float q_r_16 = q_t0_16; + float qe_15 = q_r_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dqf_16 = pool_f32[ew_1 + d_lane]; + float dkf_16 = pool_f32[ew_1 + 1024 + d_lane]; + float dgf_16 = pool_f32[ew_1 + 2048 + d_lane]; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; - float q_r_16 = q_t0_16; - float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16)) + dgf_16; + float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15))) + dgf_16; dgo_1[0] = dg_o_16; int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_284 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_316 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_285 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_317 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_33 = 16 + 8 * th + 1; @@ -3979,9 +4183,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_17 = __uint_as_float(r_303[0] << 16); float q_t0_17 = ((t_33 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1 + 128 + d_lane]; - float dkf_17 = pool_f32[ew_1 + 1152 + d_lane]; - float dgf_17 = pool_f32[ew_1 + 2176 + d_lane]; float bt_t_17 = btv_1[1]; float e_off_17 = approx_exp2(g_t_17 - gn_1); float e_dg_17 = approx_exp2(g_t_17 - gc_5); @@ -3989,22 +4190,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_17 = __fmaf_rn(ko_1[1], e_off_17, kdg_1[1] * e_dg_17); dbp_1[1] = dk2_17 * k_t_17; float dk2b_17 = dk2_17 * bt_t_17; + float q_r_17 = q_t0_17; + float qe_16 = q_r_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dqf_17 = pool_f32[ew_1 + 128 + d_lane]; + float dkf_17 = pool_f32[ew_1 + 1152 + d_lane]; + float dgf_17 = pool_f32[ew_1 + 2176 + d_lane]; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; - float q_r_17 = q_t0_17; - float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17)) + dgf_17; + float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16))) + dgf_17; dgo_1[1] = dg_o_17; int qoff_17 = t_33 * qrs; if (t_33 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_34 = 16 + 8 * th + 2; @@ -4018,9 +4227,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_18 = __uint_as_float(r_306[0] << 16); float q_t0_18 = ((t_34 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1 + 256 + d_lane]; - float dkf_18 = pool_f32[ew_1 + 1280 + d_lane]; - float dgf_18 = pool_f32[ew_1 + 2304 + d_lane]; float bt_t_18 = btv_1[2]; float e_off_18 = approx_exp2(g_t_18 - gn_1); float e_dg_18 = approx_exp2(g_t_18 - gc_5); @@ -4028,22 +4234,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_18 = __fmaf_rn(ko_1[2], e_off_18, kdg_1[2] * e_dg_18); dbp_1[2] = dk2_18 * k_t_18; float dk2b_18 = dk2_18 * bt_t_18; + float q_r_18 = q_t0_18; + float qe_17 = q_r_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dqf_18 = pool_f32[ew_1 + 256 + d_lane]; + float dkf_18 = pool_f32[ew_1 + 1280 + d_lane]; + float dgf_18 = pool_f32[ew_1 + 2304 + d_lane]; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; - float q_r_18 = q_t0_18; - float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18)) + dgf_18; + float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17))) + dgf_18; dgo_1[2] = dg_o_18; int qoff_18 = t_34 * qrs; if (t_34 < clen) { { - auto _gst_288 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_320 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_289 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_321 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_35 = 16 + 8 * th + 3; @@ -4057,9 +4271,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_19 = __uint_as_float(r_309[0] << 16); float q_t0_19 = ((t_35 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1 + 384 + d_lane]; - float dkf_19 = pool_f32[ew_1 + 1408 + d_lane]; - float dgf_19 = pool_f32[ew_1 + 2432 + d_lane]; float bt_t_19 = btv_1[3]; float e_off_19 = approx_exp2(g_t_19 - gn_1); float e_dg_19 = approx_exp2(g_t_19 - gc_5); @@ -4067,22 +4278,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_19 = __fmaf_rn(ko_1[3], e_off_19, kdg_1[3] * e_dg_19); dbp_1[3] = dk2_19 * k_t_19; float dk2b_19 = dk2_19 * bt_t_19; + float q_r_19 = q_t0_19; + float qe_18 = q_r_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dqf_19 = pool_f32[ew_1 + 384 + d_lane]; + float dkf_19 = pool_f32[ew_1 + 1408 + d_lane]; + float dgf_19 = pool_f32[ew_1 + 2432 + d_lane]; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; - float q_r_19 = q_t0_19; - float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19)) + dgf_19; + float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18))) + dgf_19; dgo_1[3] = dg_o_19; int qoff_19 = t_35 * qrs; if (t_35 < clen) { { - auto _gst_290 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_322 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_291 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_323 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_36 = 16 + 8 * th + 4; @@ -4096,9 +4315,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_20 = __uint_as_float(r_312[0] << 16); float q_t0_20 = ((t_36 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1 + 512 + d_lane]; - float dkf_20 = pool_f32[ew_1 + 1536 + d_lane]; - float dgf_20 = pool_f32[ew_1 + 2560 + d_lane]; float bt_t_20 = btv_1[4]; float e_off_20 = approx_exp2(g_t_20 - gn_1); float e_dg_20 = approx_exp2(g_t_20 - gc_5); @@ -4106,22 +4322,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_20 = __fmaf_rn(ko_1[4], e_off_20, kdg_1[4] * e_dg_20); dbp_1[4] = dk2_20 * k_t_20; float dk2b_20 = dk2_20 * bt_t_20; + float q_r_20 = q_t0_20; + float qe_19 = q_r_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dqf_20 = pool_f32[ew_1 + 512 + d_lane]; + float dkf_20 = pool_f32[ew_1 + 1536 + d_lane]; + float dgf_20 = pool_f32[ew_1 + 2560 + d_lane]; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; - float q_r_20 = q_t0_20; - float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20)) + dgf_20; + float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19))) + dgf_20; dgo_1[4] = dg_o_20; int qoff_20 = t_36 * qrs; if (t_36 < clen) { { - auto _gst_292 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_324 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_293 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_325 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_37 = 16 + 8 * th + 5; @@ -4135,9 +4359,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_21 = __uint_as_float(r_315[0] << 16); float q_t0_21 = ((t_37 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1 + 640 + d_lane]; - float dkf_21 = pool_f32[ew_1 + 1664 + d_lane]; - float dgf_21 = pool_f32[ew_1 + 2688 + d_lane]; float bt_t_21 = btv_1[5]; float e_off_21 = approx_exp2(g_t_21 - gn_1); float e_dg_21 = approx_exp2(g_t_21 - gc_5); @@ -4145,22 +4366,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_21 = __fmaf_rn(ko_1[5], e_off_21, kdg_1[5] * e_dg_21); dbp_1[5] = dk2_21 * k_t_21; float dk2b_21 = dk2_21 * bt_t_21; + float q_r_21 = q_t0_21; + float qe_20 = q_r_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dqf_21 = pool_f32[ew_1 + 640 + d_lane]; + float dkf_21 = pool_f32[ew_1 + 1664 + d_lane]; + float dgf_21 = pool_f32[ew_1 + 2688 + d_lane]; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; - float q_r_21 = q_t0_21; - float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21)) + dgf_21; + float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20))) + dgf_21; dgo_1[5] = dg_o_21; int qoff_21 = t_37 * qrs; if (t_37 < clen) { { - auto _gst_294 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_326 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_295 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_327 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_38 = 16 + 8 * th + 6; @@ -4174,9 +4403,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_22 = __uint_as_float(r_318[0] << 16); float q_t0_22 = ((t_38 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1 + 768 + d_lane]; - float dkf_22 = pool_f32[ew_1 + 1792 + d_lane]; - float dgf_22 = pool_f32[ew_1 + 2816 + d_lane]; float bt_t_22 = btv_1[6]; float e_off_22 = approx_exp2(g_t_22 - gn_1); float e_dg_22 = approx_exp2(g_t_22 - gc_5); @@ -4184,22 +4410,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_22 = __fmaf_rn(ko_1[6], e_off_22, kdg_1[6] * e_dg_22); dbp_1[6] = dk2_22 * k_t_22; float dk2b_22 = dk2_22 * bt_t_22; + float q_r_22 = q_t0_22; + float qe_21 = q_r_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dqf_22 = pool_f32[ew_1 + 768 + d_lane]; + float dkf_22 = pool_f32[ew_1 + 1792 + d_lane]; + float dgf_22 = pool_f32[ew_1 + 2816 + d_lane]; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; - float q_r_22 = q_t0_22; - float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22)) + dgf_22; + float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21))) + dgf_22; dgo_1[6] = dg_o_22; int qoff_22 = t_38 * qrs; if (t_38 < clen) { { - auto _gst_296 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_328 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_297 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_329 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_39 = 16 + 8 * th + 7; @@ -4213,9 +4447,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_23 = __uint_as_float(r_321[0] << 16); float q_t0_23 = ((t_39 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1 + 896 + d_lane]; - float dkf_23 = pool_f32[ew_1 + 1920 + d_lane]; - float dgf_23 = pool_f32[ew_1 + 2944 + d_lane]; float bt_t_23 = btv_1[7]; float e_off_23 = approx_exp2(g_t_23 - gn_1); float e_dg_23 = approx_exp2(g_t_23 - gc_5); @@ -4223,22 +4454,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_23 = __fmaf_rn(ko_1[7], e_off_23, kdg_1[7] * e_dg_23); dbp_1[7] = dk2_23 * k_t_23; float dk2b_23 = dk2_23 * bt_t_23; + float q_r_23 = q_t0_23; + float qe_22 = q_r_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dqf_23 = pool_f32[ew_1 + 896 + d_lane]; + float dkf_23 = pool_f32[ew_1 + 1920 + d_lane]; + float dgf_23 = pool_f32[ew_1 + 2944 + d_lane]; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; - float q_r_23 = q_t0_23; - float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23)) + dgf_23; + float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22))) + dgf_23; dgo_1[7] = dg_o_23; int qoff_23 = t_39 * qrs; if (t_39 < clen) { { - auto _gst_298 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_330 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_299 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_331 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_23; } } float s_run_1[1]; @@ -4267,37 +4506,37 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_332; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_333; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_334; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_335; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_336; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_337; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4315,9 +4554,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_16 = offset_1 + sfx_1[0]; if (t_40 < clen) { { - auto _gst_310 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_16; + auto _gst_342 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_16; } } int t_41 = 16 + 8 * th + 1; @@ -4325,9 +4564,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_17 = offset_1 + sfx_1[1]; if (t_41 < clen) { { - auto _gst_311 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_17; + auto _gst_343 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_17; } } int t_42 = 16 + 8 * th + 2; @@ -4335,9 +4574,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_18 = offset_1 + sfx_1[2]; if (t_42 < clen) { { - auto _gst_312 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_18; + auto _gst_344 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_18; } } int t_43 = 16 + 8 * th + 3; @@ -4345,9 +4584,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_19 = offset_1 + sfx_1[3]; if (t_43 < clen) { { - auto _gst_313 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_19; + auto _gst_345 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_19; } } int t_44 = 16 + 8 * th + 4; @@ -4355,9 +4594,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_20 = offset_1 + sfx_1[4]; if (t_44 < clen) { { - auto _gst_314 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_20; + auto _gst_346 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_20; } } int t_45 = 16 + 8 * th + 5; @@ -4365,9 +4604,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_21 = offset_1 + sfx_1[5]; if (t_45 < clen) { { - auto _gst_315 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_21; + auto _gst_347 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_21; } } int t_46 = 16 + 8 * th + 6; @@ -4375,9 +4614,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_22 = offset_1 + sfx_1[6]; if (t_46 < clen) { { - auto _gst_316 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_22; + auto _gst_348 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_22; } } int t_47 = 16 + 8 * th + 7; @@ -4385,15 +4624,15 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_23 = offset_1 + sfx_1[7]; if (t_47 < clen) { { - auto _gst_317 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_23; + auto _gst_349 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4434,9 +4673,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_24 = __uint_as_float(r_325[0] << 16); float q_t0_24 = ((t_48 >= clen) ? 0.0f : q_t_24); - float dqf_24 = pool_f32[ew_2 + d_lane]; - float dkf_24 = pool_f32[ew_2 + 1024 + d_lane]; - float dgf_24 = pool_f32[ew_2 + 2048 + d_lane]; float bt_t_24 = btv_2[0]; float e_off_24 = approx_exp2(g_t_24 - gn_2); float e_dg_24 = approx_exp2(g_t_24 - gc_6); @@ -4444,22 +4680,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_24 = kdg_2[0] * e_dg_24; dbp_2[0] = dk2_24 * k_t_24; float dk2b_24 = dk2_24 * bt_t_24; + float q_r_24 = q_t0_24; + float qe_23 = q_r_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dqf_24 = pool_f32[ew_2 + d_lane]; + float dkf_24 = pool_f32[ew_2 + 1024 + d_lane]; + float dgf_24 = pool_f32[ew_2 + 2048 + d_lane]; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; - float q_r_24 = q_t0_24; - float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24)) + dgf_24; + float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23))) + dgf_24; dgo_2[0] = dg_o_24; int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_319 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_351 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_320 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_352 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_49 = 8 * th + 1; @@ -4473,9 +4717,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_25 = __uint_as_float(r_328[0] << 16); float q_t0_25 = ((t_49 >= clen) ? 0.0f : q_t_25); - float dqf_25 = pool_f32[ew_2 + 128 + d_lane]; - float dkf_25 = pool_f32[ew_2 + 1152 + d_lane]; - float dgf_25 = pool_f32[ew_2 + 2176 + d_lane]; float bt_t_25 = btv_2[1]; float e_off_25 = approx_exp2(g_t_25 - gn_2); float e_dg_25 = approx_exp2(g_t_25 - gc_6); @@ -4483,22 +4724,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_25 = kdg_2[1] * e_dg_25; dbp_2[1] = dk2_25 * k_t_25; float dk2b_25 = dk2_25 * bt_t_25; + float q_r_25 = q_t0_25; + float qe_24 = q_r_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dqf_25 = pool_f32[ew_2 + 128 + d_lane]; + float dkf_25 = pool_f32[ew_2 + 1152 + d_lane]; + float dgf_25 = pool_f32[ew_2 + 2176 + d_lane]; float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; - float q_r_25 = q_t0_25; - float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25)) + dgf_25; + float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24))) + dgf_25; dgo_2[1] = dg_o_25; int qoff_25 = t_49 * qrs; if (t_49 < clen) { { - auto _gst_321 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_353 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_322 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_354 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_50 = 8 * th + 2; @@ -4512,9 +4761,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_26 = __uint_as_float(r_331[0] << 16); float q_t0_26 = ((t_50 >= clen) ? 0.0f : q_t_26); - float dqf_26 = pool_f32[ew_2 + 256 + d_lane]; - float dkf_26 = pool_f32[ew_2 + 1280 + d_lane]; - float dgf_26 = pool_f32[ew_2 + 2304 + d_lane]; float bt_t_26 = btv_2[2]; float e_off_26 = approx_exp2(g_t_26 - gn_2); float e_dg_26 = approx_exp2(g_t_26 - gc_6); @@ -4522,22 +4768,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_26 = kdg_2[2] * e_dg_26; dbp_2[2] = dk2_26 * k_t_26; float dk2b_26 = dk2_26 * bt_t_26; + float q_r_26 = q_t0_26; + float qe_25 = q_r_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dqf_26 = pool_f32[ew_2 + 256 + d_lane]; + float dkf_26 = pool_f32[ew_2 + 1280 + d_lane]; + float dgf_26 = pool_f32[ew_2 + 2304 + d_lane]; float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; - float q_r_26 = q_t0_26; - float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26)) + dgf_26; + float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25))) + dgf_26; dgo_2[2] = dg_o_26; int qoff_26 = t_50 * qrs; if (t_50 < clen) { { - auto _gst_323 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_355 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_324 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_356 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_51 = 8 * th + 3; @@ -4551,9 +4805,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_27 = __uint_as_float(r_334[0] << 16); float q_t0_27 = ((t_51 >= clen) ? 0.0f : q_t_27); - float dqf_27 = pool_f32[ew_2 + 384 + d_lane]; - float dkf_27 = pool_f32[ew_2 + 1408 + d_lane]; - float dgf_27 = pool_f32[ew_2 + 2432 + d_lane]; float bt_t_27 = btv_2[3]; float e_off_27 = approx_exp2(g_t_27 - gn_2); float e_dg_27 = approx_exp2(g_t_27 - gc_6); @@ -4561,22 +4812,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_27 = kdg_2[3] * e_dg_27; dbp_2[3] = dk2_27 * k_t_27; float dk2b_27 = dk2_27 * bt_t_27; + float q_r_27 = q_t0_27; + float qe_26 = q_r_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dqf_27 = pool_f32[ew_2 + 384 + d_lane]; + float dkf_27 = pool_f32[ew_2 + 1408 + d_lane]; + float dgf_27 = pool_f32[ew_2 + 2432 + d_lane]; float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; - float q_r_27 = q_t0_27; - float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27)) + dgf_27; + float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26))) + dgf_27; dgo_2[3] = dg_o_27; int qoff_27 = t_51 * qrs; if (t_51 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_52 = 8 * th + 4; @@ -4590,9 +4849,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_28 = __uint_as_float(r_337[0] << 16); float q_t0_28 = ((t_52 >= clen) ? 0.0f : q_t_28); - float dqf_28 = pool_f32[ew_2 + 512 + d_lane]; - float dkf_28 = pool_f32[ew_2 + 1536 + d_lane]; - float dgf_28 = pool_f32[ew_2 + 2560 + d_lane]; float bt_t_28 = btv_2[4]; float e_off_28 = approx_exp2(g_t_28 - gn_2); float e_dg_28 = approx_exp2(g_t_28 - gc_6); @@ -4600,22 +4856,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_28 = kdg_2[4] * e_dg_28; dbp_2[4] = dk2_28 * k_t_28; float dk2b_28 = dk2_28 * bt_t_28; + float q_r_28 = q_t0_28; + float qe_27 = q_r_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dqf_28 = pool_f32[ew_2 + 512 + d_lane]; + float dkf_28 = pool_f32[ew_2 + 1536 + d_lane]; + float dgf_28 = pool_f32[ew_2 + 2560 + d_lane]; float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; - float q_r_28 = q_t0_28; - float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28)) + dgf_28; + float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27))) + dgf_28; dgo_2[4] = dg_o_28; int qoff_28 = t_52 * qrs; if (t_52 < clen) { { - auto _gst_327 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_359 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_328 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_360 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_53 = 8 * th + 5; @@ -4629,9 +4893,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_29 = __uint_as_float(r_340[0] << 16); float q_t0_29 = ((t_53 >= clen) ? 0.0f : q_t_29); - float dqf_29 = pool_f32[ew_2 + 640 + d_lane]; - float dkf_29 = pool_f32[ew_2 + 1664 + d_lane]; - float dgf_29 = pool_f32[ew_2 + 2688 + d_lane]; float bt_t_29 = btv_2[5]; float e_off_29 = approx_exp2(g_t_29 - gn_2); float e_dg_29 = approx_exp2(g_t_29 - gc_6); @@ -4639,22 +4900,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_29 = kdg_2[5] * e_dg_29; dbp_2[5] = dk2_29 * k_t_29; float dk2b_29 = dk2_29 * bt_t_29; + float q_r_29 = q_t0_29; + float qe_28 = q_r_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dqf_29 = pool_f32[ew_2 + 640 + d_lane]; + float dkf_29 = pool_f32[ew_2 + 1664 + d_lane]; + float dgf_29 = pool_f32[ew_2 + 2688 + d_lane]; float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; - float q_r_29 = q_t0_29; - float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29)) + dgf_29; + float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28))) + dgf_29; dgo_2[5] = dg_o_29; int qoff_29 = t_53 * qrs; if (t_53 < clen) { { - auto _gst_329 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_361 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_330 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_362 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_54 = 8 * th + 6; @@ -4668,9 +4937,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_30 = __uint_as_float(r_343[0] << 16); float q_t0_30 = ((t_54 >= clen) ? 0.0f : q_t_30); - float dqf_30 = pool_f32[ew_2 + 768 + d_lane]; - float dkf_30 = pool_f32[ew_2 + 1792 + d_lane]; - float dgf_30 = pool_f32[ew_2 + 2816 + d_lane]; float bt_t_30 = btv_2[6]; float e_off_30 = approx_exp2(g_t_30 - gn_2); float e_dg_30 = approx_exp2(g_t_30 - gc_6); @@ -4678,22 +4944,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_30 = kdg_2[6] * e_dg_30; dbp_2[6] = dk2_30 * k_t_30; float dk2b_30 = dk2_30 * bt_t_30; + float q_r_30 = q_t0_30; + float qe_29 = q_r_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dqf_30 = pool_f32[ew_2 + 768 + d_lane]; + float dkf_30 = pool_f32[ew_2 + 1792 + d_lane]; + float dgf_30 = pool_f32[ew_2 + 2816 + d_lane]; float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; - float q_r_30 = q_t0_30; - float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30)) + dgf_30; + float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29))) + dgf_30; dgo_2[6] = dg_o_30; int qoff_30 = t_54 * qrs; if (t_54 < clen) { { - auto _gst_331 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_363 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_332 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_364 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_55 = 8 * th + 7; @@ -4707,9 +4981,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_31 = __uint_as_float(r_346[0] << 16); float q_t0_31 = ((t_55 >= clen) ? 0.0f : q_t_31); - float dqf_31 = pool_f32[ew_2 + 896 + d_lane]; - float dkf_31 = pool_f32[ew_2 + 1920 + d_lane]; - float dgf_31 = pool_f32[ew_2 + 2944 + d_lane]; float bt_t_31 = btv_2[7]; float e_off_31 = approx_exp2(g_t_31 - gn_2); float e_dg_31 = approx_exp2(g_t_31 - gc_6); @@ -4717,22 +4988,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_31 = kdg_2[7] * e_dg_31; dbp_2[7] = dk2_31 * k_t_31; float dk2b_31 = dk2_31 * bt_t_31; + float q_r_31 = q_t0_31; + float qe_30 = q_r_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dqf_31 = pool_f32[ew_2 + 896 + d_lane]; + float dkf_31 = pool_f32[ew_2 + 1920 + d_lane]; + float dgf_31 = pool_f32[ew_2 + 2944 + d_lane]; float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; - float q_r_31 = q_t0_31; - float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31)) + dgf_31; + float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30))) + dgf_31; dgo_2[7] = dg_o_31; int qoff_31 = t_55 * qrs; if (t_55 < clen) { { - auto _gst_333 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_365 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_334 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_366 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_31; } } float s_run_2[1]; @@ -4761,35 +5040,35 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_367; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_368; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_369; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_370; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_371; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_372; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_2) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4807,9 +5086,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_24 = offset_2 + sfx_2[0]; if (t_56 < clen) { { - auto _gst_345 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_24; + auto _gst_377 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_24; } } int t_57 = 8 * th + 1; @@ -4817,9 +5096,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_25 = offset_2 + sfx_2[1]; if (t_57 < clen) { { - auto _gst_346 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_25; + auto _gst_378 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_25; } } int t_58 = 8 * th + 2; @@ -4827,9 +5106,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_26 = offset_2 + sfx_2[2]; if (t_58 < clen) { { - auto _gst_347 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_26; + auto _gst_379 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_26; } } int t_59 = 8 * th + 3; @@ -4837,9 +5116,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_27 = offset_2 + sfx_2[3]; if (t_59 < clen) { { - auto _gst_348 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_27; + auto _gst_380 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_27; } } int t_60 = 8 * th + 4; @@ -4847,9 +5126,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_28 = offset_2 + sfx_2[4]; if (t_60 < clen) { { - auto _gst_349 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_28; + auto _gst_381 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_28; } } int t_61 = 8 * th + 5; @@ -4857,9 +5136,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_29 = offset_2 + sfx_2[5]; if (t_61 < clen) { { - auto _gst_350 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_29; + auto _gst_382 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_29; } } int t_62 = 8 * th + 6; @@ -4867,9 +5146,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_30 = offset_2 + sfx_2[6]; if (t_62 < clen) { { - auto _gst_351 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_30; + auto _gst_383 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_30; } } int t_63 = 8 * th + 7; @@ -4877,9 +5156,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_31 = offset_2 + sfx_2[7]; if (t_63 < clen) { { - auto _gst_352 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_31; + auto _gst_384 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_31; } } off_acc[0] = off_acc[0] + t_hi_2 + t_lo_2; diff --git a/flashinfer/experimental/cake_kda_det_train/build.json b/flashinfer/experimental/cake_kda_det_train/build.json index 20f13b8c718..6d353295534 100644 --- a/flashinfer/experimental/cake_kda_det_train/build.json +++ b/flashinfer/experimental/cake_kda_det_train/build.json @@ -55,9 +55,9 @@ } }, "generator": { - "cake_revision": "3462767011d1e0de9c84de41bbb90397fa1ebafb", + "cake_revision": "66dfdbba90b203e45aa6f05fb399f056cf525863", "cuda": "13.4.92", - "rendered_at": "2026-11-04T08:37:59Z" + "rendered_at": "2026-11-05T07:57:50Z" }, "libraries": [ { @@ -128,7 +128,7 @@ "closure_sha256": "be78da9cf92f6390c7abce6079b23a01c20b0c417e88182f0a9dfa56899f12ea", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_036e1c7e1dfc95a88aae_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -191,7 +191,7 @@ "closure_sha256": "01f3800933989cb908e2d41cd308d5f1299e7c60b67b135fcd6b6872055cb482", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_058a68a5b4d89c53be4b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -842,7 +842,7 @@ "closure_sha256": "36e389cfe8bb3e87d3283b12f2ca6344d53605eb8f9ccd77e971630e669e98a9", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_1f9c46f98f4290ee26df_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -968,7 +968,7 @@ "closure_sha256": "e2adf85405786dba245894592541739d1dfd8b13d9b7b2559a3f3673a51a9221", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_22812d87c26b53a1ba10_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1031,7 +1031,7 @@ "closure_sha256": "290ff01688dfddc78f3b3aedf13ca2b8f3cb3a9d5a2031611d20888a16fb1916", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_22a2f04d41df18554b84_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1094,7 +1094,7 @@ "closure_sha256": "e169902bb5fffbab95950d7fd24116403986df3abb89fd1190a802c96013de7f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_243fb0b23aea1b296ab6_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1362,75 +1362,12 @@ "csrc/cake_kda_det_train_3162ed758e5ddfb97c17_binding.cu" ] }, - { - "arch": "sm_100a", - "closure_sha256": "d67635fb51a6027711e2f5ce62e5b0989cd9209e4c40f2c87e39f26d9ef72cc5", - "entry": "run", - "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", - "name": "cake_kda_det_train_328afe43951979e798d1_sm_100a", - "nvcc_flags": [ - "--use_fast_math" - ], - "nvcc_flags_if": [], - "path": "aot/cake_kda_det_train_328afe43951979e798d1_sm_100a.so", - "sha256": { - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu": "a2280dbb0317696d0de63cfcc1f058d72f078434209069346fe7226985ca1a4f", - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu": "6280fa5f6602c176fa9302f54db505440868f4646457059095c217bdb73a5c74" - }, - "sources": [ - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu", - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu" - ] - }, - { - "arch": "sm_103a", - "closure_sha256": "5f6a438c7ab175156bc9b6904cf18003c88d9f7ece6e9ccf0ca270d96de27d39", - "entry": "run", - "gencode": "-gencode=arch=compute_103a,code=sm_103a", - "min_cuda": "12.9", - "name": "cake_kda_det_train_328afe43951979e798d1_sm_103a", - "nvcc_flags": [ - "--use_fast_math" - ], - "nvcc_flags_if": [], - "path": "aot/cake_kda_det_train_328afe43951979e798d1_sm_103a.so", - "sha256": { - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu": "a2280dbb0317696d0de63cfcc1f058d72f078434209069346fe7226985ca1a4f", - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu": "6280fa5f6602c176fa9302f54db505440868f4646457059095c217bdb73a5c74" - }, - "sources": [ - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu", - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu" - ] - }, - { - "arch": "sm_107a", - "closure_sha256": "22287ac196b49a8a2197bc12e46031243498844c97cfb992c3698d1f406f97c9", - "entry": "run", - "gencode": "-gencode=arch=compute_107a,code=sm_107a", - "min_cuda": "13.4", - "name": "cake_kda_det_train_328afe43951979e798d1_sm_107a", - "nvcc_flags": [ - "--use_fast_math" - ], - "nvcc_flags_if": [], - "path": "aot/cake_kda_det_train_328afe43951979e798d1_sm_107a.so", - "sha256": { - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu": "a2280dbb0317696d0de63cfcc1f058d72f078434209069346fe7226985ca1a4f", - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu": "6280fa5f6602c176fa9302f54db505440868f4646457059095c217bdb73a5c74" - }, - "sources": [ - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu", - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu" - ] - }, { "arch": "sm_100a", "closure_sha256": "fb9a5b5ed6a3a643c231bff18c0274bae18ad4d1a309773bebc176decb9997bb", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_3712054f19d4180caa28_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1490,7 +1427,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "b92a651f508dcd7fcb456899a611dee82b426afc34ce35d00b0367bcb90478e1", + "closure_sha256": "690c6d305fa5613747ebc52a8fb7b73a214d7afb2f1f64d82e7a02dc4ca6a1d2", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1502,7 +1439,7 @@ "path": "aot/cake_kda_det_train_37ed4f705c286adadd40_sm_100a.so", "sha256": { "csrc/cake_kda_det_train_37ed4f705c286adadd40_binding.cu": "5202f3619a6ff62aafe1a71c3cdef4d087ee3980a6dbc17eaf6c786fdc3863af", - "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "bb39cb550bf2117abedd25f0b15f341d7cb5d3d7a773a15b75990284e1c949d1" + "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "5e333a7df6481cbce675eff02a232727b5cfcf243413f9951a31f46e15c21aa5" }, "sources": [ "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu", @@ -1511,7 +1448,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "a9cc65744f8f52c242100d916a0aa15b95a541cda910fe406030b7731f24b248", + "closure_sha256": "079af35e3f5d255d74875bdb7e15cb689f0ab701c720dc74ddb449630ea112b7", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -1523,7 +1460,7 @@ "path": "aot/cake_kda_det_train_37ed4f705c286adadd40_sm_103a.so", "sha256": { "csrc/cake_kda_det_train_37ed4f705c286adadd40_binding.cu": "5202f3619a6ff62aafe1a71c3cdef4d087ee3980a6dbc17eaf6c786fdc3863af", - "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "bb39cb550bf2117abedd25f0b15f341d7cb5d3d7a773a15b75990284e1c949d1" + "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "5e333a7df6481cbce675eff02a232727b5cfcf243413f9951a31f46e15c21aa5" }, "sources": [ "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu", @@ -1532,7 +1469,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "06a0b4e168e5f7cd47a1ffcb025998be5820d28735b4237acb5b907ef0d35ccc", + "closure_sha256": "b2481688ed2c449f22d38ba7dde8dab602ee01e9c5dbdcaeb19a8a989e82065d", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -1544,7 +1481,7 @@ "path": "aot/cake_kda_det_train_37ed4f705c286adadd40_sm_107a.so", "sha256": { "csrc/cake_kda_det_train_37ed4f705c286adadd40_binding.cu": "5202f3619a6ff62aafe1a71c3cdef4d087ee3980a6dbc17eaf6c786fdc3863af", - "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "bb39cb550bf2117abedd25f0b15f341d7cb5d3d7a773a15b75990284e1c949d1" + "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "5e333a7df6481cbce675eff02a232727b5cfcf243413f9951a31f46e15c21aa5" }, "sources": [ "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu", @@ -1640,7 +1577,7 @@ "closure_sha256": "e47e8b8db302377a0fa3f1e57ffed611dad307b53fabc4babb91780de18be8d1", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_3f4b037a0a1f8dcc314b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1829,7 +1766,7 @@ "closure_sha256": "85e7e25b555d9017a2e3b18fbcb3d3c3a9d99dfdb183e1b34ba860ac52c24c62", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_4734a5d4f63d815ec086_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1976,7 +1913,7 @@ "closure_sha256": "ad53bdf6062f0005d086416f3425a74249b1d5e21ef02bda36f4c917eeed3ecf", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_4cce3cc5bc6210c95ce3_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2078,7 +2015,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "3609084337ae5d9a546310b69bb2fb9fb834e1a485d1703015703957c312f1e1", + "closure_sha256": "c5cd38315e7557fa15c790fbbc886457ab7103527e2619c3a77d953d4dc22422", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -2090,7 +2027,7 @@ "path": "aot/cake_kda_det_train_54786ffd01bd9b6bb416_sm_100a.so", "sha256": { "csrc/cake_kda_det_train_54786ffd01bd9b6bb416_binding.cu": "5ce7e802c6b0993c88a0902e1e01a19510157db4b3af37238eb4f07ef7cff3bf", - "csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu": "f4c9f7508fd777c791df07e6a872ec959ea0efa4c655a64eec155092001e1491" + "csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu": "ec6ddd0feb25279017baa490321b21987af12940f8886b9f0c8c77326488b6dd" }, "sources": [ "csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu", @@ -2165,7 +2102,7 @@ "closure_sha256": "d5c36ecaff1392e62c22eda675af29ac282f1403e598f0b5090ea967fd4a3f01", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_5f5a5b3715675e083823_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2228,7 +2165,7 @@ "closure_sha256": "91178a228e0de03ef823845d47c1bf808a0d5d911203ce2963ef29688b26af5d", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_5fcc76904e5e2dd891de_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2351,7 +2288,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "588999d7dd184af482d50c8dcc2bbe11ad4e63d3d8d5412575b1d8753a29e323", + "closure_sha256": "173b5af1082b4ee723c7515b4de34b3c33c2be0a4c9ac4815e3c32f763581452", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -2363,7 +2300,7 @@ "path": "aot/cake_kda_det_train_6308ef19a5bf0691fcfe_sm_100a.so", "sha256": { "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_binding.cu": "443d9bd452639c361d50f955da2c34a0127adac99eefecf25fd3ee876f614cf4", - "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "e8d8e01623afc65774a41e7a6dfe45993b15f6b27a11d01c27179c76b5b5db7a" + "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "03331aeb5c9eec8b79f370946cdf424e7405a4644c9ac87afb8d23932749f70b" }, "sources": [ "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu", @@ -2372,7 +2309,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "584e6cc20076b734cb00e9b8b5013ce66bd6a60780520c6eec61ffcee87354ad", + "closure_sha256": "ff0a8f4669a4732c15b65a97d42c004cae2cabca76ac8cbb93b11bf94d223f8d", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -2384,7 +2321,7 @@ "path": "aot/cake_kda_det_train_6308ef19a5bf0691fcfe_sm_103a.so", "sha256": { "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_binding.cu": "443d9bd452639c361d50f955da2c34a0127adac99eefecf25fd3ee876f614cf4", - "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "e8d8e01623afc65774a41e7a6dfe45993b15f6b27a11d01c27179c76b5b5db7a" + "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "03331aeb5c9eec8b79f370946cdf424e7405a4644c9ac87afb8d23932749f70b" }, "sources": [ "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu", @@ -2393,7 +2330,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "ddfee41a698e7899e10c3ea88bd0240c46f72f51acfb8e80b1f35662e9db6b4b", + "closure_sha256": "8bbd0bacf61e59235f1efe3b6017f8246374d82c81b5511bb291dd402d751149", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -2405,7 +2342,7 @@ "path": "aot/cake_kda_det_train_6308ef19a5bf0691fcfe_sm_107a.so", "sha256": { "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_binding.cu": "443d9bd452639c361d50f955da2c34a0127adac99eefecf25fd3ee876f614cf4", - "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "e8d8e01623afc65774a41e7a6dfe45993b15f6b27a11d01c27179c76b5b5db7a" + "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "03331aeb5c9eec8b79f370946cdf424e7405a4644c9ac87afb8d23932749f70b" }, "sources": [ "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu", @@ -2417,7 +2354,7 @@ "closure_sha256": "d0677903f801235cc735df70a4571e4882e3cc034c6ff7de2ac25a6c8edbcf2a", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_706240be56d4e41df1a2_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2732,7 +2669,7 @@ "closure_sha256": "f1548497bb1c39094345d5e758e50544837610c16dd8a5aff7c385f83195952f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_7eca60b36e233943dae6_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3047,7 +2984,7 @@ "closure_sha256": "cf5e6f417b179ed37d478f879334ff5e82403160993eb2100d7a67310f9b1e5c", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_898a1eb972fbaf55789a_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3110,7 +3047,7 @@ "closure_sha256": "86bc258e380620ba45b64d1fe29d21b829fb1983741d571e3e8aebf3699109e5", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_8ab0929b8154c90b5eff_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3425,7 +3362,7 @@ "closure_sha256": "8c570885d73c86a9054988b8b792cb95c00a507716ed78a11513bc42e1b4a984", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_909a0aa57efb3f353972_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3551,7 +3488,7 @@ "closure_sha256": "7dbb4035f2c693851c8c7f15824acc32b42d3c65a5036ccecda66da5dd87b23f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_945a87fa84a265fc9f94_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3740,7 +3677,7 @@ "closure_sha256": "6ac7a7e3d9cf4572c7069a549540381c74954b1356fff208cd3713772e7e16e7", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_9931cdff5dfd86c687c7_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3800,23 +3737,23 @@ }, { "arch": "sm_103a", - "closure_sha256": "28ccc6594e50f48ab26dde04d911c06fbf31988a14c65b54a863284097d3aecc", + "closure_sha256": "543e22ef8c1b9f331328cccba1a27d95f33255cfcd1ff13343a0b0406356f00a", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_kda_det_train_9ad80f98f22e0b3bce08_sm_103a", + "name": "cake_kda_det_train_a135a53a639f33f921ab_sm_103a", "nvcc_flags": [ "--use_fast_math" ], "nvcc_flags_if": [], - "path": "aot/cake_kda_det_train_9ad80f98f22e0b3bce08_sm_103a.so", + "path": "aot/cake_kda_det_train_a135a53a639f33f921ab_sm_103a.so", "sha256": { - "csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_binding.cu": "f93536d86b23d345a48d1061d31ab92fdf34358df70f0556c636f5e607a1cd2b", - "csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_kernel.cu": "85eec7d912d6a8512f86898f7653bfe532642223c57276612937fb2a3fdf9588" + "csrc/cake_kda_det_train_a135a53a639f33f921ab_binding.cu": "dd0fd6333712a0d413c21180afc4d1c5e67eecb746f2764b27430256944cfd20", + "csrc/cake_kda_det_train_a135a53a639f33f921ab_kernel.cu": "76603465d1aec9fc51afd4cf116acbcbb2d7f74da58d421b26ba276405b8ec62" }, "sources": [ - "csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_kernel.cu", - "csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_binding.cu" + "csrc/cake_kda_det_train_a135a53a639f33f921ab_kernel.cu", + "csrc/cake_kda_det_train_a135a53a639f33f921ab_binding.cu" ] }, { @@ -3887,7 +3824,7 @@ "closure_sha256": "f48e9fc3ec2e1e6ea7367e9c68bb1d65222329ab95a2f311ee5e21b85953a0b1", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_a557058a8aeca49a6402_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4010,7 +3947,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "774a1be99b4d9cace2b218eac26278924aa81ab1250a605caef0778fc1f30f5d", + "closure_sha256": "8f3c67b0b027fe7f712268d6a2cfef75990908ea9cff633093a1d2187bdf6b76", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4022,7 +3959,7 @@ "path": "aot/cake_kda_det_train_aa6b5c23c97e1cb9880a_sm_107a.so", "sha256": { "csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_binding.cu": "debc670b1148d75a7a7a5921f82594f94e87c6a0e3164c8d96e78d72f40109a1", - "csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu": "298916e699ec308a3f6f915e493b8fa4fc32773ed05b570ac8309d559922eb28" + "csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu": "fb9a573e5e64c79c62009e3d468b443e61dd36ab4f5b3f010e80835f01718ebf" }, "sources": [ "csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu", @@ -4034,7 +3971,7 @@ "closure_sha256": "0d270842881dfb70f68fa4a8e648163de22f0b7cfa96e436080d5d10f5017a36", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_ac974e171d56fc3ce10b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4092,6 +4029,69 @@ "csrc/cake_kda_det_train_ac974e171d56fc3ce10b_binding.cu" ] }, + { + "arch": "sm_100a", + "closure_sha256": "b829f5bb6544edac23cd576e8fd10299eaa224a9a1a99c498f4da58a5a0aa7a3", + "entry": "run", + "gencode": "-gencode=arch=compute_100a,code=sm_100a", + "min_cuda": "12.8", + "name": "cake_kda_det_train_ad4c93dbb0435933c8a1_sm_100a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_kda_det_train_ad4c93dbb0435933c8a1_sm_100a.so", + "sha256": { + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu": "d764d721a4dee7c1711eb31a083621008300ecca4d848e6605f52382ef19860d", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu": "89771fb71e1e717a0a76dc466148d1d1ae551911872e5c22f2446694f780190d" + }, + "sources": [ + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu" + ] + }, + { + "arch": "sm_103a", + "closure_sha256": "b4888425f74e8c7feea2070856dd688f91ba57b2473a13ab093b0df48867f403", + "entry": "run", + "gencode": "-gencode=arch=compute_103a,code=sm_103a", + "min_cuda": "12.9", + "name": "cake_kda_det_train_ad4c93dbb0435933c8a1_sm_103a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_kda_det_train_ad4c93dbb0435933c8a1_sm_103a.so", + "sha256": { + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu": "d764d721a4dee7c1711eb31a083621008300ecca4d848e6605f52382ef19860d", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu": "89771fb71e1e717a0a76dc466148d1d1ae551911872e5c22f2446694f780190d" + }, + "sources": [ + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "0a6eb50f3f56ad769b58ea39f24e31ac71de1d0e48823c22d5df425f45d6ec9c", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_kda_det_train_ad4c93dbb0435933c8a1_sm_107a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_kda_det_train_ad4c93dbb0435933c8a1_sm_107a.so", + "sha256": { + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu": "d764d721a4dee7c1711eb31a083621008300ecca4d848e6605f52382ef19860d", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu": "89771fb71e1e717a0a76dc466148d1d1ae551911872e5c22f2446694f780190d" + }, + "sources": [ + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu" + ] + }, { "arch": "sm_100a", "closure_sha256": "fedf5e76ae0bd20e07ba2faafa288acd00f225fdd7f8c4ea3274cd2f2b2710fc", @@ -4157,7 +4157,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "e53c26da2f9396fb00c28e0b1026af4b6bf7eb15b26510621698ff24e8562a5f", + "closure_sha256": "a96205205c8b3f8c81383cb0b2a05ac0c8df59a6665fd14a15ec557639ef6fb7", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -4169,7 +4169,7 @@ "path": "aot/cake_kda_det_train_b16a0c1729083004fe80_sm_100a.so", "sha256": { "csrc/cake_kda_det_train_b16a0c1729083004fe80_binding.cu": "794bfda5c0f38f682e0e47f7cbe493ba7259e8e3452cbf57e7ae8418774bd19c", - "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "97168c857930c9c0b958c53c9fc454e22f3db5466d9e6662605b4365c7e7373b" + "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "0a7461ee10c6fc594a467a600d7db7440f2bcaa32ceee0defacf08fb67695d5c" }, "sources": [ "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu", @@ -4178,7 +4178,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "73dc6915cf91d10ee0f7cf64edb39915c25a5f718b42068332cdf87495812157", + "closure_sha256": "c580e4dc184c0e79fa5d3da8d3c78e5b1c230acce2bfdfac73a44daa66458c42", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -4190,7 +4190,7 @@ "path": "aot/cake_kda_det_train_b16a0c1729083004fe80_sm_103a.so", "sha256": { "csrc/cake_kda_det_train_b16a0c1729083004fe80_binding.cu": "794bfda5c0f38f682e0e47f7cbe493ba7259e8e3452cbf57e7ae8418774bd19c", - "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "97168c857930c9c0b958c53c9fc454e22f3db5466d9e6662605b4365c7e7373b" + "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "0a7461ee10c6fc594a467a600d7db7440f2bcaa32ceee0defacf08fb67695d5c" }, "sources": [ "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu", @@ -4199,7 +4199,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "6c4d32517a7f05553af5004f8de4be2ecc7c36406d99645e4f33920d79245a4d", + "closure_sha256": "8fcf6c93f1fae0283afe853ae3aa670854c39cd542c853339037098de5d9e934", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4211,7 +4211,7 @@ "path": "aot/cake_kda_det_train_b16a0c1729083004fe80_sm_107a.so", "sha256": { "csrc/cake_kda_det_train_b16a0c1729083004fe80_binding.cu": "794bfda5c0f38f682e0e47f7cbe493ba7259e8e3452cbf57e7ae8418774bd19c", - "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "97168c857930c9c0b958c53c9fc454e22f3db5466d9e6662605b4365c7e7373b" + "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "0a7461ee10c6fc594a467a600d7db7440f2bcaa32ceee0defacf08fb67695d5c" }, "sources": [ "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu", @@ -4346,7 +4346,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "f47aad030860d78edaab9dc981a976aa4eacf304d6f575a12916910ae9e367d5", + "closure_sha256": "d3f05c11435bea7db383e301a1fd650f028a37a6b72cc37230234e04eae2b2c9", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -4358,7 +4358,7 @@ "path": "aot/cake_kda_det_train_c27b09eb5f5881be2969_sm_103a.so", "sha256": { "csrc/cake_kda_det_train_c27b09eb5f5881be2969_binding.cu": "d9d3b2beefd5de36931ceef887eab30fc410aa5a7de2179528b93dbb32992b5a", - "csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu": "b4c04f2c996190ba3f750aa6193ddcbc0c89a560da3c324f06cbaa223bd0169e" + "csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu": "97de6f775d8944105d6690adce2ac575693d3c7eb7b704a556a95e4b1659030f" }, "sources": [ "csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu", @@ -4622,7 +4622,7 @@ "closure_sha256": "08b1943d3b751202f87afd37d466ef82f4161f19dadbaeddfecec3ca744373d8", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_d32455627b87ba0a1d87_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4811,7 +4811,7 @@ "closure_sha256": "c64f17ffdd12bee3c4a4e1b930d146e6d7abde8a3802e40d039ef99887613ac9", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_dec885b973bd44f88416_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -5063,7 +5063,7 @@ "closure_sha256": "10c66cfb894c37ef5f81e7cda989d7fa3b50558497c449d6414861afa81c3201", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_e60e7ca5d88192193e61_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -5186,7 +5186,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "a5f2c4044a4aceacd6d92955aa224bcd6463933cd8db00955969090ef63736c5", + "closure_sha256": "ccde9d8566ae05a6a9679a92c3671c0f31dea64236c85e247213e23e78564134", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -5198,7 +5198,7 @@ "path": "aot/cake_kda_det_train_ea14e16d1fc71364ba03_sm_100a.so", "sha256": { "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_binding.cu": "252a5ddb33acb0fc2a321354f365d5dd61560c0852e5c673f3b5c3a2fa181782", - "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "a18014739476a9a46b0357eb2cef236b99058427cf3dd8aeb58f0f85adb82951" + "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "376a2d7b861c534f2b41f5748cd4fab0f8993d8b21c0c645949f8859ce11c498" }, "sources": [ "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu", @@ -5207,7 +5207,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "4129ba9d2edb5707676fea6497f4f5e0cc844ccab4788eb4d919fb6290c4f6d8", + "closure_sha256": "2e5a195be353b3251ead917e0fc7db239c4326907b64f8ee1281cbf7e69e2357", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -5219,7 +5219,7 @@ "path": "aot/cake_kda_det_train_ea14e16d1fc71364ba03_sm_103a.so", "sha256": { "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_binding.cu": "252a5ddb33acb0fc2a321354f365d5dd61560c0852e5c673f3b5c3a2fa181782", - "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "a18014739476a9a46b0357eb2cef236b99058427cf3dd8aeb58f0f85adb82951" + "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "376a2d7b861c534f2b41f5748cd4fab0f8993d8b21c0c645949f8859ce11c498" }, "sources": [ "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu", @@ -5228,7 +5228,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "4645d75bb22547eeb8e78f7aada81a738f63103ee460296d859b3315e055d291", + "closure_sha256": "e83e9937787c1620b61ee3ca1500045b6762880ef3aa7d18b9e06b449150ba0e", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -5240,7 +5240,7 @@ "path": "aot/cake_kda_det_train_ea14e16d1fc71364ba03_sm_107a.so", "sha256": { "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_binding.cu": "252a5ddb33acb0fc2a321354f365d5dd61560c0852e5c673f3b5c3a2fa181782", - "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "a18014739476a9a46b0357eb2cef236b99058427cf3dd8aeb58f0f85adb82951" + "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "376a2d7b861c534f2b41f5748cd4fab0f8993d8b21c0c645949f8859ce11c498" }, "sources": [ "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu", @@ -5315,7 +5315,7 @@ "closure_sha256": "b30e9a22c26b09d6b4561307e4525f8f6a90944caa71989462e3624edd5ab86a", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_eac26fffb3df8780302c_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -5378,7 +5378,7 @@ "closure_sha256": "05ec3212fb39e9b1e8d81a29c8e6b5ff101bfc3c74a98f34ace71d031401ebb2", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_f011cd9add8f3c7c3eee_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -5504,7 +5504,7 @@ "closure_sha256": "925f0e20f2aa1d2b2e4eff092eaf4552651afa913c5d9a86d518deedd9d7ccc2", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_f6a70fb0288a508f2828_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -5630,7 +5630,7 @@ "closure_sha256": "f8a8577c36e3970ab36ec45b8f0686e2522826e35ac4b6227ce9fcf508e220e6", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_f92b256cccc2c9007554_sm_100a", "nvcc_flags": [ "--use_fast_math" diff --git a/flashinfer/experimental/cake_kda_det_train/cake_jit.py b/flashinfer/experimental/cake_kda_det_train/cake_jit.py index 21e7ad9bd04..8e12a828e10 100644 --- a/flashinfer/experimental/cake_kda_det_train/cake_jit.py +++ b/flashinfer/experimental/cake_kda_det_train/cake_jit.py @@ -7366,7 +7366,7 @@ ], "ffi_entry": "run", "grid_rule": {"kind": "chunks", "x_factor": 1}, - "kernel_symbol": "kernel_cake_kda_det_train_328afe43951979e798d1", + "kernel_symbol": "kernel_cake_kda_det_train_ad4c93dbb0435933c8a1", "l2n": ["fused"], "launch": { "block": (288, 1, 1), @@ -7375,7 +7375,7 @@ "dynamic_smem_bytes": 115200, "use_pdl": False, }, - "library": "cake_kda_det_train_328afe43951979e798d1", + "library": "cake_kda_det_train_ad4c93dbb0435933c8a1", "logical": { "beta": "beta_int", "chunk_bos": "chunk_bos", @@ -7457,7 +7457,7 @@ ], "ffi_entry": "run", "grid_rule": {"kind": "chunks", "x_factor": 1}, - "kernel_symbol": "kernel_cake_kda_det_train_9ad80f98f22e0b3bce08", + "kernel_symbol": "kernel_cake_kda_det_train_a135a53a639f33f921ab", "l2n": ["fused"], "launch": { "block": (288, 1, 1), @@ -7466,7 +7466,7 @@ "dynamic_smem_bytes": 115200, "use_pdl": False, }, - "library": "cake_kda_det_train_9ad80f98f22e0b3bce08", + "library": "cake_kda_det_train_a135a53a639f33f921ab", "logical": { "beta": "beta_int", "chunk_bos": "chunk_bos", diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu index 9db0e86d2f0..0239f524cb4 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2249,12 +2272,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_3 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * e2_3); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_3, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_3 : __fmaf_rn(dl, _rcp_170, e2_3)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2278,12 +2307,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_4 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * e2_4); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_4, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_4, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_4 : __fmaf_rn(dl_1, _rcp_174, e2_4)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2307,12 +2342,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_5 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * e2_5); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_5, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_5, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_5 : __fmaf_rn(dl_3, _rcp_178, e2_5)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2336,12 +2377,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_6 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * e2_6); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_6, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_6, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_6 : __fmaf_rn(dl_5, _rcp_182, e2_6)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_2 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_1 = approx_exp2(dc2_2.x); float e2_1_1 = approx_exp2(dc2_2.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_190 = approx_rcp(k_t0_2); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_191 = approx_rcp(k_t1_2); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_2.x; pan_641[3] = p2_2.y; int t0_3 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_2 = approx_exp2(dc2_3.x); float e2_1_2 = approx_exp2(dc2_3.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_194 = approx_rcp(k_t0_3); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_195 = approx_rcp(k_t1_3); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_3.x; pan_641[5] = p2_3.y; int t0_4 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(_f2_61, make_float2(g_t0_4, g_t1_4)); float e2_0_3 = approx_exp2(dc2_4.x); float e2_1_3 = approx_exp2(dc2_4.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_198 = approx_rcp(k_t0_4); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_199 = approx_rcp(k_t1_4); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_4.x; pan_641[7] = p2_4.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2505,12 +2576,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_202 = approx_rcp(k_t0_5); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_203 = approx_rcp(k_t1_5); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_5.x; pan_703[1] = p2_5.y; int t0_6 = 8 * th + 2; @@ -2534,12 +2611,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_2.x); float en_1_1 = approx_exp2(dn2_2.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_206 = approx_rcp(k_t0_6); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_207 = approx_rcp(k_t1_6); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_6.x; pan_703[3] = p2_6.y; int t0_7 = 8 * th + 4; @@ -2563,12 +2646,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_3.x); float en_1_2 = approx_exp2(dn2_3.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_210 = approx_rcp(k_t0_7); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_211 = approx_rcp(k_t1_7); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_7.x; pan_703[5] = p2_7.y; int t0_8 = 8 * th + 6; @@ -2592,12 +2681,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_4.x); float en_1_3 = approx_exp2(dn2_4.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_214 = approx_rcp(k_t0_8); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_215 = approx_rcp(k_t1_8); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_8.x; pan_703[7] = p2_8.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2639,12 +2734,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_5.x); float en_1_4 = approx_exp2(dn2_5.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_218 = approx_rcp(k_t0_9); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_219 = approx_rcp(k_t1_9); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_9.x; pan_777[1] = p2_9.y; int t0_10 = 16 + 8 * th + 2; @@ -2668,12 +2769,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_6.x); float en_1_5 = approx_exp2(dn2_6.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_222 = approx_rcp(k_t0_10); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_223 = approx_rcp(k_t1_10); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_10.x; pan_777[3] = p2_10.y; int t0_11 = 16 + 8 * th + 4; @@ -2697,12 +2804,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_7.x); float en_1_6 = approx_exp2(dn2_7.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_226 = approx_rcp(k_t0_11); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_227 = approx_rcp(k_t1_11); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_11.x; pan_777[5] = p2_11.y; int t0_12 = 16 + 8 * th + 6; @@ -2726,12 +2839,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_8 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_8.x); float en_1_7 = approx_exp2(dn2_8.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_230 = approx_rcp(k_t0_12); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_231 = approx_rcp(k_t1_12); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_12.x; pan_777[7] = p2_12.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2802,37 +2921,37 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_4 = approx_exp2(d2_57.x); float e1_4 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_4, e1_4); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_4, e1_4); float2 v2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_4.x; v_562_0[1] = v2_4.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_5 = approx_exp2(d2_58.x); float e1_5 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_5, e1_5); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_5, e1_5); float2 v2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_5.x; v_562_0[3] = v2_5.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_6 = approx_exp2(d2_59.x); float e1_6 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_6, e1_6); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_6, e1_6); float2 v2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_6.x; v_562_0[5] = v2_6.y; float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_7 = approx_exp2(d2_60.x); float e1_7 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_7, e1_7); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_7, e1_7); float2 v2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_7.x; v_562_0[7] = v2_7.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2966,7 +3085,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -2998,7 +3122,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3030,7 +3159,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_2)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3062,7 +3196,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_3)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3094,7 +3233,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_4)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3126,7 +3270,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_5)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3158,7 +3307,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_6)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3190,7 +3344,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_7)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3205,31 +3364,31 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_216 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_216; + float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_248; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_217 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_217; + float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_249; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_218 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_218; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_250; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_219 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_219; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_251; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_220 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_220; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_252; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_221 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_221; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_253; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_222 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_222; - float _shfl_xor_sync_223 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_223; - float _shfl_xor_sync_224 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_224; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_254; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_255; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_256; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); if (32 + 8 * th < clen) { @@ -3337,7 +3496,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; + float qe_7 = q_t_1_8 * e_dg_0; + float kp_19 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_0; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_14[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_0) * _tmem_load_15[7]); + float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_16[7], k_t_0, cr_8)) + dgf_0; s_run_1059[0] = s_run_1059[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3369,7 +3533,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[6]; - float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, (dk2b_2 - _tmem_load_16[6]) * k_t_2) + dgf_2; + float qe_8 = q_t_1_9 * e_dg_2; + float kp_20 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_14[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_2) * _tmem_load_15[6]); + float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_16[6], k_t_2, cr_9)) + dgf_2; s_run_1059[0] = s_run_1059[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3401,7 +3570,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[5]; - float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, (dk2b_3 - _tmem_load_16[5]) * k_t_3) + dgf_3; + float qe_9 = q_t_1_10 * e_dg_3; + float kp_21 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_3; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_14[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_3) * _tmem_load_15[5]); + float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_16[5], k_t_3, cr_10)) + dgf_3; s_run_1059[0] = s_run_1059[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3433,7 +3607,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[4]; - float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, (dk2b_4 - _tmem_load_16[4]) * k_t_4) + dgf_4; + float qe_10 = q_t_1_11 * e_dg_4; + float kp_22 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_4; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_14[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_4) * _tmem_load_15[4]); + float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_16[4], k_t_4, cr_11)) + dgf_4; s_run_1059[0] = s_run_1059[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3465,7 +3644,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[3]; - float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, (dk2b_5 - _tmem_load_16[3]) * k_t_5) + dgf_5; + float qe_11 = q_t_1_12 * e_dg_5; + float kp_23 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_5; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_14[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_5) * _tmem_load_15[3]); + float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_16[3], k_t_5, cr_12)) + dgf_5; s_run_1059[0] = s_run_1059[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3497,7 +3681,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[2]; - float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, (dk2b_6 - _tmem_load_16[2]) * k_t_6) + dgf_6; + float qe_12 = q_t_1_13 * e_dg_6; + float kp_24 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_6; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_14[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_6) * _tmem_load_15[2]); + float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_16[2], k_t_6, cr_13)) + dgf_6; s_run_1059[0] = s_run_1059[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3529,7 +3718,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[1]; - float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, (dk2b_7 - _tmem_load_16[1]) * k_t_7) + dgf_7; + float qe_13 = q_t_1_14 * e_dg_7; + float kp_25 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_7; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_14[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_7) * _tmem_load_15[1]); + float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_16[1], k_t_7, cr_14)) + dgf_7; s_run_1059[0] = s_run_1059[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3561,7 +3755,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_16[0]; - float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, (dk2b_8 - _tmem_load_16[0]) * k_t_8) + dgf_8; + float qe_14 = q_t_1_15 * e_dg_8; + float kp_26 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_8; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_14[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_8) * _tmem_load_15[0]); + float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_16[0], k_t_8, cr_15)) + dgf_8; s_run_1059[0] = s_run_1059[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3576,31 +3775,31 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int b2_1231 = lane_1 >> 2 & 1; float v4_1232[4]; float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_225 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_225; + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_257; float send_2 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_226 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_226; + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_258; float send_3 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_227 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_227; + float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_259; float send_4 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_228 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_228; + float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_260; float v2_1237[2]; float send_5 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_229 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_229; + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_261; float send_6 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_230 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_230; + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); + v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_262; float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_231 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_231; - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_232; - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_233; + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); + float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_263; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); + v1_1241 = v1_1241 + _shfl_xor_sync_264; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); + v1_1241 = v1_1241 + _shfl_xor_sync_265; asm volatile("tcgen05.fence::before_thread_sync;"); if (48 + 8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + (48 + 8 * th)) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(98304 - 32768 * th >> 2) + d_lane]; @@ -3652,8 +3851,8 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM } mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_234 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_234), "f"(tot_1228) : "memory"); + uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_1228) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3682,9 +3881,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_0 = offset + _tmem_load_17[0]; if (t_9 < clen) { { - auto _gst_235 = dg_base + goff_1_0; - __builtin_assume(__isGlobal(_gst_235)); - *(reinterpret_cast(_gst_235) + (0)) = dg_1_0; + auto _gst_267 = dg_base + goff_1_0; + __builtin_assume(__isGlobal(_gst_267)); + *(reinterpret_cast(_gst_267) + (0)) = dg_1_0; } } int t_10 = 48 + 8 * th + 1; @@ -3692,9 +3891,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_1 = offset + _tmem_load_17[1]; if (t_10 < clen) { { - auto _gst_236 = dg_base + goff_1_1; - __builtin_assume(__isGlobal(_gst_236)); - *(reinterpret_cast(_gst_236) + (0)) = dg_1_1; + auto _gst_268 = dg_base + goff_1_1; + __builtin_assume(__isGlobal(_gst_268)); + *(reinterpret_cast(_gst_268) + (0)) = dg_1_1; } } int t_11 = 48 + 8 * th + 2; @@ -3702,9 +3901,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_2 = offset + _tmem_load_17[2]; if (t_11 < clen) { { - auto _gst_237 = dg_base + goff_1_2; - __builtin_assume(__isGlobal(_gst_237)); - *(reinterpret_cast(_gst_237) + (0)) = dg_1_2; + auto _gst_269 = dg_base + goff_1_2; + __builtin_assume(__isGlobal(_gst_269)); + *(reinterpret_cast(_gst_269) + (0)) = dg_1_2; } } int t_12 = 48 + 8 * th + 3; @@ -3712,9 +3911,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_3 = offset + _tmem_load_17[3]; if (t_12 < clen) { { - auto _gst_238 = dg_base + goff_1_3; - __builtin_assume(__isGlobal(_gst_238)); - *(reinterpret_cast(_gst_238) + (0)) = dg_1_3; + auto _gst_270 = dg_base + goff_1_3; + __builtin_assume(__isGlobal(_gst_270)); + *(reinterpret_cast(_gst_270) + (0)) = dg_1_3; } } int t_13 = 48 + 8 * th + 4; @@ -3722,9 +3921,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_4 = offset + _tmem_load_17[4]; if (t_13 < clen) { { - auto _gst_239 = dg_base + goff_1_4; - __builtin_assume(__isGlobal(_gst_239)); - *(reinterpret_cast(_gst_239) + (0)) = dg_1_4; + auto _gst_271 = dg_base + goff_1_4; + __builtin_assume(__isGlobal(_gst_271)); + *(reinterpret_cast(_gst_271) + (0)) = dg_1_4; } } int t_14 = 48 + 8 * th + 5; @@ -3732,9 +3931,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_5 = offset + _tmem_load_17[5]; if (t_14 < clen) { { - auto _gst_240 = dg_base + goff_1_5; - __builtin_assume(__isGlobal(_gst_240)); - *(reinterpret_cast(_gst_240) + (0)) = dg_1_5; + auto _gst_272 = dg_base + goff_1_5; + __builtin_assume(__isGlobal(_gst_272)); + *(reinterpret_cast(_gst_272) + (0)) = dg_1_5; } } int t_15 = 48 + 8 * th + 6; @@ -3742,9 +3941,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_6 = offset + _tmem_load_17[6]; if (t_15 < clen) { { - auto _gst_241 = dg_base + goff_1_6; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast(_gst_241) + (0)) = dg_1_6; + auto _gst_273 = dg_base + goff_1_6; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast(_gst_273) + (0)) = dg_1_6; } } int t_16 = 48 + 8 * th + 7; @@ -3752,9 +3951,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_7 = offset + _tmem_load_17[7]; if (t_16 < clen) { { - auto _gst_242 = dg_base + goff_1_7; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast(_gst_242) + (0)) = dg_1_7; + auto _gst_274 = dg_base + goff_1_7; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast(_gst_274) + (0)) = dg_1_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3772,9 +3971,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_0 = offset_1250 + _tmem_load_19[0]; if (t_17 < clen) { { - auto _gst_243 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast(_gst_243) + (0)) = dg_0; + auto _gst_275 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast(_gst_275) + (0)) = dg_0; } } int t_18 = 32 + 8 * th + 1; @@ -3782,9 +3981,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1 = offset_1250 + _tmem_load_19[1]; if (t_18 < clen) { { - auto _gst_244 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast(_gst_244) + (0)) = dg_1; + auto _gst_276 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast(_gst_276) + (0)) = dg_1; } } int t_19 = 32 + 8 * th + 2; @@ -3792,9 +3991,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_2 = offset_1250 + _tmem_load_19[2]; if (t_19 < clen) { { - auto _gst_245 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast(_gst_245) + (0)) = dg_2; + auto _gst_277 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast(_gst_277) + (0)) = dg_2; } } int t_20 = 32 + 8 * th + 3; @@ -3802,9 +4001,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_3 = offset_1250 + _tmem_load_19[3]; if (t_20 < clen) { { - auto _gst_246 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast(_gst_246) + (0)) = dg_3; + auto _gst_278 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast(_gst_278) + (0)) = dg_3; } } int t_21 = 32 + 8 * th + 4; @@ -3812,9 +4011,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_4 = offset_1250 + _tmem_load_19[4]; if (t_21 < clen) { { - auto _gst_247 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast(_gst_247) + (0)) = dg_4; + auto _gst_279 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast(_gst_279) + (0)) = dg_4; } } int t_22 = 32 + 8 * th + 5; @@ -3822,9 +4021,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_5 = offset_1250 + _tmem_load_19[5]; if (t_22 < clen) { { - auto _gst_248 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast(_gst_248) + (0)) = dg_5; + auto _gst_280 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast(_gst_280) + (0)) = dg_5; } } int t_23 = 32 + 8 * th + 6; @@ -3832,9 +4031,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_6 = offset_1250 + _tmem_load_19[6]; if (t_23 < clen) { { - auto _gst_249 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast(_gst_249) + (0)) = dg_6; + auto _gst_281 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast(_gst_281) + (0)) = dg_6; } } int t_24 = 32 + 8 * th + 7; @@ -3842,15 +4041,15 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_7 = offset_1250 + _tmem_load_19[7]; if (t_24 < clen) { { - auto _gst_250 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast(_gst_250) + (0)) = dg_7; + auto _gst_282 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast(_gst_282) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi_1248 + t_lo_1249; { - uint32_t _addr_251 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_251), "f"(off_acc[0]) : "memory"); + uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3910,7 +4109,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[7]; - float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, (dk2b_9 - _tmem_load_24[7]) * k_t_9) + dgf_9; + float qe_15 = q_t_1_16 * e_dg_9; + float kp_27 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_9; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_9) * _tmem_load_23[7]); + float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_24[7], k_t_9, cr_16)) + dgf_9; s_run_1302[0] = s_run_1302[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3942,7 +4146,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[6]; - float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, (dk2b_10 - _tmem_load_24[6]) * k_t_10) + dgf_10; + float qe_16 = q_t_1_17 * e_dg_10; + float kp_28 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_10; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_10) * _tmem_load_23[6]); + float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_24[6], k_t_10, cr_17)) + dgf_10; s_run_1302[0] = s_run_1302[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3974,7 +4183,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[5]; - float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, (dk2b_11 - _tmem_load_24[5]) * k_t_11) + dgf_11; + float qe_17 = q_t_1_18 * e_dg_11; + float kp_29 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_11; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_11) * _tmem_load_23[5]); + float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_24[5], k_t_11, cr_18)) + dgf_11; s_run_1302[0] = s_run_1302[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4006,7 +4220,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[4]; - float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, (dk2b_12 - _tmem_load_24[4]) * k_t_12) + dgf_12; + float qe_18 = q_t_1_19 * e_dg_12; + float kp_30 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_12; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_12) * _tmem_load_23[4]); + float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_24[4], k_t_12, cr_19)) + dgf_12; s_run_1302[0] = s_run_1302[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4038,7 +4257,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[3]; - float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, (dk2b_13 - _tmem_load_24[3]) * k_t_13) + dgf_13; + float qe_19 = q_t_1_20 * e_dg_13; + float kp_31 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_13; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_13) * _tmem_load_23[3]); + float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_24[3], k_t_13, cr_20)) + dgf_13; s_run_1302[0] = s_run_1302[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4070,7 +4294,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[2]; - float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, (dk2b_14 - _tmem_load_24[2]) * k_t_14) + dgf_14; + float qe_20 = q_t_1_21 * e_dg_14; + float kp_32 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_14; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_14) * _tmem_load_23[2]); + float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_24[2], k_t_14, cr_21)) + dgf_14; s_run_1302[0] = s_run_1302[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4102,7 +4331,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[1]; - float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, (dk2b_15 - _tmem_load_24[1]) * k_t_15) + dgf_15; + float qe_21 = q_t_1_22 * e_dg_15; + float kp_33 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_15; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_15) * _tmem_load_23[1]); + float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_24[1], k_t_15, cr_22)) + dgf_15; s_run_1302[0] = s_run_1302[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4134,7 +4368,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_24[0]; - float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, (dk2b_16 - _tmem_load_24[0]) * k_t_16) + dgf_16; + float qe_22 = q_t_1_23 * e_dg_16; + float kp_34 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_16; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_16) * _tmem_load_23[0]); + float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, __fmaf_rn(dk2b_16 - _tmem_load_24[0], k_t_16, cr_23)) + dgf_16; s_run_1302[0] = s_run_1302[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4149,31 +4388,31 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; float send_7 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_252; + float _shfl_xor_sync_284 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_284; float send_8 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_253; + float _shfl_xor_sync_285 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_285; float send_9 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_254; + float _shfl_xor_sync_286 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_286; float send_10 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_255; + float _shfl_xor_sync_287 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_287; float v2_1480[2]; float send_11 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_256; + float _shfl_xor_sync_288 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_288; float send_12 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_289; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_258; - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_259; - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_260; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_290; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_291; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_292; asm volatile("tcgen05.fence::before_thread_sync;"); if (16 + 8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + (16 + 8 * th)) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(81920 - 32768 * th >> 2) + d_lane]; @@ -4226,8 +4465,8 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_261 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_261), "f"(tot_1471) : "memory"); + uint32_t _addr_293 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_293), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4256,9 +4495,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_8 = offset_1488 + _tmem_load_25[0]; if (t_33 < clen) { { - auto _gst_262 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_262)); - *(reinterpret_cast(_gst_262) + (0)) = dg_8; + auto _gst_294 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_294)); + *(reinterpret_cast(_gst_294) + (0)) = dg_8; } } int t_34 = 16 + 8 * th + 1; @@ -4266,9 +4505,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_9 = offset_1488 + _tmem_load_25[1]; if (t_34 < clen) { { - auto _gst_263 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_263)); - *(reinterpret_cast(_gst_263) + (0)) = dg_9; + auto _gst_295 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_295)); + *(reinterpret_cast(_gst_295) + (0)) = dg_9; } } int t_35 = 16 + 8 * th + 2; @@ -4276,9 +4515,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_10 = offset_1488 + _tmem_load_25[2]; if (t_35 < clen) { { - auto _gst_264 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_264)); - *(reinterpret_cast(_gst_264) + (0)) = dg_10; + auto _gst_296 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_296)); + *(reinterpret_cast(_gst_296) + (0)) = dg_10; } } int t_36 = 16 + 8 * th + 3; @@ -4286,9 +4525,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_11 = offset_1488 + _tmem_load_25[3]; if (t_36 < clen) { { - auto _gst_265 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_265)); - *(reinterpret_cast(_gst_265) + (0)) = dg_11; + auto _gst_297 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_297)); + *(reinterpret_cast(_gst_297) + (0)) = dg_11; } } int t_37 = 16 + 8 * th + 4; @@ -4296,9 +4535,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_12 = offset_1488 + _tmem_load_25[4]; if (t_37 < clen) { { - auto _gst_266 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_266)); - *(reinterpret_cast(_gst_266) + (0)) = dg_12; + auto _gst_298 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_298)); + *(reinterpret_cast(_gst_298) + (0)) = dg_12; } } int t_38 = 16 + 8 * th + 5; @@ -4306,9 +4545,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_13 = offset_1488 + _tmem_load_25[5]; if (t_38 < clen) { { - auto _gst_267 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_13; + auto _gst_299 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_13; } } int t_39 = 16 + 8 * th + 6; @@ -4316,9 +4555,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_14 = offset_1488 + _tmem_load_25[6]; if (t_39 < clen) { { - auto _gst_268 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_14; + auto _gst_300 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_14; } } int t_40 = 16 + 8 * th + 7; @@ -4326,15 +4565,15 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_15 = offset_1488 + _tmem_load_25[7]; if (t_40 < clen) { { - auto _gst_269 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_15; + auto _gst_301 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_1486 + t_lo_1487; { - uint32_t _addr_270 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_270), "f"(off_acc[0]) : "memory"); + uint32_t _addr_302 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_302), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4388,7 +4627,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[7]; - float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, (dk2b_17 - _tmem_load_28[7]) * k_t_17) + dgf_17; + float qe_23 = q_t_1_24 * e_dg_17; + float kp_35 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_35; + float kbe_23 = (float)kb_23 * e_dg_17; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_35 * e_dg_17) * _tmem_load_27[7]); + float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, __fmaf_rn(dk2b_17 - _tmem_load_28[7], k_t_17, cr_24)) + dgf_17; s_run_1524[0] = s_run_1524[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4420,7 +4664,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[6]; - float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, (dk2b_18 - _tmem_load_28[6]) * k_t_18) + dgf_18; + float qe_24 = q_t_1_25 * e_dg_18; + float kp_36 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_36; + float kbe_24 = (float)kb_24 * e_dg_18; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_36 * e_dg_18) * _tmem_load_27[6]); + float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, __fmaf_rn(dk2b_18 - _tmem_load_28[6], k_t_18, cr_25)) + dgf_18; s_run_1524[0] = s_run_1524[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4452,7 +4701,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[5]; - float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, (dk2b_19 - _tmem_load_28[5]) * k_t_19) + dgf_19; + float qe_25 = q_t_1_26 * e_dg_19; + float kp_37 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_37; + float kbe_25 = (float)kb_25 * e_dg_19; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_37 * e_dg_19) * _tmem_load_27[5]); + float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, __fmaf_rn(dk2b_19 - _tmem_load_28[5], k_t_19, cr_26)) + dgf_19; s_run_1524[0] = s_run_1524[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4484,7 +4738,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[4]; - float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, (dk2b_20 - _tmem_load_28[4]) * k_t_20) + dgf_20; + float qe_26 = q_t_1_27 * e_dg_20; + float kp_38 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_38; + float kbe_26 = (float)kb_26 * e_dg_20; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_38 * e_dg_20) * _tmem_load_27[4]); + float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, __fmaf_rn(dk2b_20 - _tmem_load_28[4], k_t_20, cr_27)) + dgf_20; s_run_1524[0] = s_run_1524[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4516,7 +4775,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[3]; - float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, (dk2b_21 - _tmem_load_28[3]) * k_t_21) + dgf_21; + float qe_27 = q_t_1_28 * e_dg_21; + float kp_39 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_39; + float kbe_27 = (float)kb_27 * e_dg_21; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_39 * e_dg_21) * _tmem_load_27[3]); + float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, __fmaf_rn(dk2b_21 - _tmem_load_28[3], k_t_21, cr_28)) + dgf_21; s_run_1524[0] = s_run_1524[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4548,7 +4812,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[2]; - float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, (dk2b_22 - _tmem_load_28[2]) * k_t_22) + dgf_22; + float qe_28 = q_t_1_29 * e_dg_22; + float kp_40 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_40; + float kbe_28 = (float)kb_28 * e_dg_22; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_40 * e_dg_22) * _tmem_load_27[2]); + float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, __fmaf_rn(dk2b_22 - _tmem_load_28[2], k_t_22, cr_29)) + dgf_22; s_run_1524[0] = s_run_1524[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4580,7 +4849,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[1]; - float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, (dk2b_23 - _tmem_load_28[1]) * k_t_23) + dgf_23; + float qe_29 = q_t_1_30 * e_dg_23; + float kp_41 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_41; + float kbe_29 = (float)kb_29 * e_dg_23; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_41 * e_dg_23) * _tmem_load_27[1]); + float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, __fmaf_rn(dk2b_23 - _tmem_load_28[1], k_t_23, cr_30)) + dgf_23; s_run_1524[0] = s_run_1524[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4612,7 +4886,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + _tmem_load_28[0]; - float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, (dk2b_24 - _tmem_load_28[0]) * k_t_24) + dgf_24; + float qe_30 = q_t_1_31 * e_dg_24; + float kp_42 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_42; + float kbe_30 = (float)kb_30 * e_dg_24; + float cr_31 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_42 * e_dg_24) * _tmem_load_27[0]); + float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, __fmaf_rn(dk2b_24 - _tmem_load_28[0], k_t_24, cr_31)) + dgf_24; s_run_1524[0] = s_run_1524[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4627,31 +4906,31 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; float send_13 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_271; + float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_303; float send_14 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_272; + float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_304; float send_15 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_273; + float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_305; float send_16 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_274; + float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_306; float v2_1702[2]; float send_17 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_275; + float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_307; float send_18 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_276; + float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_308; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_277; - float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_278; - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_279; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_309; + float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_310; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_311; if (8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + 8 * th) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(98304 - 32768 * th >> 2) + d_lane]; } @@ -4702,8 +4981,8 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM } asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_280 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_280), "f"(tot_1693) : "memory"); + uint32_t _addr_312 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_312), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4732,9 +5011,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_16 = offset_1710 + _tmem_load_29[0]; if (t_49 < clen) { { - auto _gst_281 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_16; + auto _gst_313 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_16; } } int t_50 = 8 * th + 1; @@ -4742,9 +5021,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_17 = offset_1710 + _tmem_load_29[1]; if (t_50 < clen) { { - auto _gst_282 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_17; + auto _gst_314 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_17; } } int t_51 = 8 * th + 2; @@ -4752,9 +5031,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_18 = offset_1710 + _tmem_load_29[2]; if (t_51 < clen) { { - auto _gst_283 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_283)); - *(reinterpret_cast(_gst_283) + (0)) = dg_18; + auto _gst_315 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_315)); + *(reinterpret_cast(_gst_315) + (0)) = dg_18; } } int t_52 = 8 * th + 3; @@ -4762,9 +5041,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_19 = offset_1710 + _tmem_load_29[3]; if (t_52 < clen) { { - auto _gst_284 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast(_gst_284) + (0)) = dg_19; + auto _gst_316 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = dg_19; } } int t_53 = 8 * th + 4; @@ -4772,9 +5051,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_20 = offset_1710 + _tmem_load_29[4]; if (t_53 < clen) { { - auto _gst_285 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast(_gst_285) + (0)) = dg_20; + auto _gst_317 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast(_gst_317) + (0)) = dg_20; } } int t_54 = 8 * th + 5; @@ -4782,9 +5061,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_21 = offset_1710 + _tmem_load_29[5]; if (t_54 < clen) { { - auto _gst_286 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast(_gst_286) + (0)) = dg_21; + auto _gst_318 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = dg_21; } } int t_55 = 8 * th + 6; @@ -4792,9 +5071,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_22 = offset_1710 + _tmem_load_29[6]; if (t_55 < clen) { { - auto _gst_287 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast(_gst_287) + (0)) = dg_22; + auto _gst_319 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast(_gst_319) + (0)) = dg_22; } } int t_56 = 8 * th + 7; @@ -4802,9 +5081,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_23 = offset_1710 + _tmem_load_29[7]; if (t_56 < clen) { { - auto _gst_288 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast(_gst_288) + (0)) = dg_23; + auto _gst_320 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1708 + t_lo_1709; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu index 3f76831b164..cc293b1672b 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu @@ -130,6 +130,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -172,6 +179,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2235,15 +2258,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2264,15 +2293,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2293,15 +2328,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2322,15 +2363,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2366,12 +2413,18 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2391,12 +2444,18 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2416,12 +2475,18 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2441,12 +2506,18 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2491,15 +2562,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2520,15 +2597,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2549,15 +2632,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2578,15 +2667,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2625,15 +2720,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2654,15 +2755,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2683,15 +2790,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2712,15 +2825,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2791,37 +2910,37 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2833,29 +2952,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2863,29 +2982,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2953,24 +3072,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; - float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0) + dgf_0; + float qe = q_t0_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr)) + dgf_0; int goff_0 = t_0 * rs_g; int qoff_0 = t_0 * qrs; { - auto _gst_216 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast(_gst_216) + (0)) = dg_o_0; + auto _gst_248 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast(_gst_248) + (0)) = dg_o_0; } if (t_0 < clen) { { - auto _gst_217 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_249 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_218 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_250 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -2996,24 +3120,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_1 = dk2_1 * bt_t_1; float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; - float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1) + dgf_1; + float qe_0 = q_t0_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0)) + dgf_1; int goff_1 = t_1 * rs_g; int qoff_1 = t_1 * qrs; { - auto _gst_219 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast(_gst_219) + (0)) = dg_o_1; + auto _gst_251 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast(_gst_251) + (0)) = dg_o_1; } if (t_1 < clen) { { - auto _gst_220 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_252 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_221 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_253 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3039,24 +3168,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; - float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2) + dgf_2; + float qe_1 = q_t0_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1)) + dgf_2; int goff_2 = t_2 * rs_g; int qoff_2 = t_2 * qrs; { - auto _gst_222 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast(_gst_222) + (0)) = dg_o_2; + auto _gst_254 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast(_gst_254) + (0)) = dg_o_2; } if (t_2 < clen) { { - auto _gst_223 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_255 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_224 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_256 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3082,24 +3216,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; - float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3) + dgf_3; + float qe_2 = q_t0_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2)) + dgf_3; int goff_3 = t_3 * rs_g; int qoff_3 = t_3 * qrs; { - auto _gst_225 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast(_gst_225) + (0)) = dg_o_3; + auto _gst_257 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast(_gst_257) + (0)) = dg_o_3; } if (t_3 < clen) { { - auto _gst_226 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_258 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_227 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_259 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3125,24 +3264,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; - float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4) + dgf_4; + float qe_3 = q_t0_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3)) + dgf_4; int goff_4 = t_4 * rs_g; int qoff_4 = t_4 * qrs; { - auto _gst_228 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast(_gst_228) + (0)) = dg_o_4; + auto _gst_260 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast(_gst_260) + (0)) = dg_o_4; } if (t_4 < clen) { { - auto _gst_229 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_261 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_230 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_262 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3168,24 +3312,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; - float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5) + dgf_5; + float qe_4 = q_t0_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4)) + dgf_5; int goff_5 = t_5 * rs_g; int qoff_5 = t_5 * qrs; { - auto _gst_231 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast(_gst_231) + (0)) = dg_o_5; + auto _gst_263 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast(_gst_263) + (0)) = dg_o_5; } if (t_5 < clen) { { - auto _gst_232 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_232)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_232) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_264 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_264)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_264) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_233 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_233)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_233) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_265 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_265)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3211,24 +3360,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; - float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6) + dgf_6; + float qe_5 = q_t0_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5)) + dgf_6; int goff_6 = t_6 * rs_g; int qoff_6 = t_6 * qrs; { - auto _gst_234 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_234)); - *(reinterpret_cast(_gst_234) + (0)) = dg_o_6; + auto _gst_266 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_266)); + *(reinterpret_cast(_gst_266) + (0)) = dg_o_6; } if (t_6 < clen) { { - auto _gst_235 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_235)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_235) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_267 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_267)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_267) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_236 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_236)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_236) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_268 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_268)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3254,24 +3408,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; - float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7) + dgf_7; + float qe_6 = q_t0_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6)) + dgf_7; int goff_7 = t_7 * rs_g; int qoff_7 = t_7 * qrs; { - auto _gst_237 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_237)); - *(reinterpret_cast(_gst_237) + (0)) = dg_o_7; + auto _gst_269 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_269)); + *(reinterpret_cast(_gst_269) + (0)) = dg_o_7; } if (t_7 < clen) { { - auto _gst_238 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_238)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_238) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_270 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_270)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_270) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_239 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_239)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_239) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_271 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_271)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dk_o_7; } } int b4 = lane_1 >> 4 & 1; @@ -3279,31 +3438,31 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_240; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_272; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_241 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_241; + float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_273; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_242 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_242; + float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_274; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_243 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_243; + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_275; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_244 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_244; + float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_276; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_245 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_245; + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_277; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_246 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_246; - float _shfl_xor_sync_247 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_247; - float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_248; + float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_278; + float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_279; + float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_280; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3361,24 +3520,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; - float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8) + dgf_8; + float qe_7 = q_t0_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7)) + dgf_8; int goff_8 = t_8 * rs_g; int qoff_8 = t_8 * qrs; { - auto _gst_249 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast(_gst_249) + (0)) = dg_o_8; + auto _gst_281 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast(_gst_281) + (0)) = dg_o_8; } if (t_8 < clen) { { - auto _gst_250 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_282 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_251 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_283 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3404,24 +3568,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; - float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9) + dgf_9; + float qe_8 = q_t0_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8)) + dgf_9; int goff_9 = t_9 * rs_g; int qoff_9 = t_9 * qrs; { - auto _gst_252 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast(_gst_252) + (0)) = dg_o_9; + auto _gst_284 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast(_gst_284) + (0)) = dg_o_9; } if (t_9 < clen) { { - auto _gst_253 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_285 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_254 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_286 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3447,24 +3616,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; - float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10) + dgf_10; + float qe_9 = q_t0_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9)) + dgf_10; int goff_10 = t_10 * rs_g; int qoff_10 = t_10 * qrs; { - auto _gst_255 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast(_gst_255) + (0)) = dg_o_10; + auto _gst_287 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast(_gst_287) + (0)) = dg_o_10; } if (t_10 < clen) { { - auto _gst_256 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_288 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_257 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_257)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_289 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_289)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3490,24 +3664,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; - float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11) + dgf_11; + float qe_10 = q_t0_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10)) + dgf_11; int goff_11 = t_11 * rs_g; int qoff_11 = t_11 * qrs; { - auto _gst_258 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_258)); - *(reinterpret_cast(_gst_258) + (0)) = dg_o_11; + auto _gst_290 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_290)); + *(reinterpret_cast(_gst_290) + (0)) = dg_o_11; } if (t_11 < clen) { { - auto _gst_259 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_259)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_291 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_291)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_260 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_260)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_292 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_292)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3533,24 +3712,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; - float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12) + dgf_12; + float qe_11 = q_t0_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11)) + dgf_12; int goff_12 = t_12 * rs_g; int qoff_12 = t_12 * qrs; { - auto _gst_261 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_261)); - *(reinterpret_cast(_gst_261) + (0)) = dg_o_12; + auto _gst_293 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_293)); + *(reinterpret_cast(_gst_293) + (0)) = dg_o_12; } if (t_12 < clen) { { - auto _gst_262 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_262)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_294 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_294)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_263 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_263)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_295 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_295)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3576,24 +3760,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; - float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13) + dgf_13; + float qe_12 = q_t0_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12)) + dgf_13; int goff_13 = t_13 * rs_g; int qoff_13 = t_13 * qrs; { - auto _gst_264 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_264)); - *(reinterpret_cast(_gst_264) + (0)) = dg_o_13; + auto _gst_296 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_296)); + *(reinterpret_cast(_gst_296) + (0)) = dg_o_13; } if (t_13 < clen) { { - auto _gst_265 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_265)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_297 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_297)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_266 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_266)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_266) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_298 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_298)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3619,24 +3808,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; - float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14) + dgf_14; + float qe_13 = q_t0_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13)) + dgf_14; int goff_14 = t_14 * rs_g; int qoff_14 = t_14 * qrs; { - auto _gst_267 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_o_14; + auto _gst_299 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_o_14; } if (t_14 < clen) { { - auto _gst_268 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_300 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_300) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_269 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_269) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_301 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3662,24 +3856,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; - float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15) + dgf_15; + float qe_14 = q_t0_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14)) + dgf_15; int goff_15 = t_15 * rs_g; int qoff_15 = t_15 * qrs; { - auto _gst_270 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_o_15; + auto _gst_302 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_o_15; } if (t_15 < clen) { { - auto _gst_271 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_303 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_303) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_272 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_272) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_304 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dk_o_15; } } int b4_0 = lane_1 >> 4 & 1; @@ -3687,31 +3886,31 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_273; + float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_305; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_274; + float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_306; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_275; + float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_307; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_276; + float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_308; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_277; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_309; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_278; + float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_310; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_279; - float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_280; - float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_281; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_311; + float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_312; + float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_313; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); mbarrier_wait(p1b_ready_addr, 0); @@ -3770,24 +3969,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; - float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16) + dgf_16; + float qe_15 = q_t0_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15)) + dgf_16; int goff_16 = t_16 * rs_g; int qoff_16 = t_16 * qrs; { - auto _gst_282 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_o_16; + auto _gst_314 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_o_16; } if (t_16 < clen) { { - auto _gst_283 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_283)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_315 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_315)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_315) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_284 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_316 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_17 = 16 + 8 * th + 1; @@ -3813,24 +4017,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; - float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17) + dgf_17; + float qe_16 = q_t0_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16)) + dgf_17; int goff_17 = t_17 * rs_g; int qoff_17 = t_17 * qrs; { - auto _gst_285 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast(_gst_285) + (0)) = dg_o_17; + auto _gst_317 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast(_gst_317) + (0)) = dg_o_17; } if (t_17 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_18 = 16 + 8 * th + 2; @@ -3856,24 +4065,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; - float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18) + dgf_18; + float qe_17 = q_t0_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17)) + dgf_18; int goff_18 = t_18 * rs_g; int qoff_18 = t_18 * qrs; { - auto _gst_288 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast(_gst_288) + (0)) = dg_o_18; + auto _gst_320 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = dg_o_18; } if (t_18 < clen) { { - auto _gst_289 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_321 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_290 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_322 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_19 = 16 + 8 * th + 3; @@ -3899,24 +4113,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; - float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19) + dgf_19; + float qe_18 = q_t0_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18)) + dgf_19; int goff_19 = t_19 * rs_g; int qoff_19 = t_19 * qrs; { - auto _gst_291 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast(_gst_291) + (0)) = dg_o_19; + auto _gst_323 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast(_gst_323) + (0)) = dg_o_19; } if (t_19 < clen) { { - auto _gst_292 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_324 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_293 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_325 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_20 = 16 + 8 * th + 4; @@ -3942,24 +4161,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; - float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20) + dgf_20; + float qe_19 = q_t0_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19)) + dgf_20; int goff_20 = t_20 * rs_g; int qoff_20 = t_20 * qrs; { - auto _gst_294 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast(_gst_294) + (0)) = dg_o_20; + auto _gst_326 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast(_gst_326) + (0)) = dg_o_20; } if (t_20 < clen) { { - auto _gst_295 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_327 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_296 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_328 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_21 = 16 + 8 * th + 5; @@ -3985,24 +4209,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; - float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21) + dgf_21; + float qe_20 = q_t0_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20)) + dgf_21; int goff_21 = t_21 * rs_g; int qoff_21 = t_21 * qrs; { - auto _gst_297 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast(_gst_297) + (0)) = dg_o_21; + auto _gst_329 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast(_gst_329) + (0)) = dg_o_21; } if (t_21 < clen) { { - auto _gst_298 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_330 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_299 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_331 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_22 = 16 + 8 * th + 6; @@ -4028,24 +4257,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; - float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22) + dgf_22; + float qe_21 = q_t0_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21)) + dgf_22; int goff_22 = t_22 * rs_g; int qoff_22 = t_22 * qrs; { - auto _gst_300 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_300)); - *(reinterpret_cast(_gst_300) + (0)) = dg_o_22; + auto _gst_332 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_332)); + *(reinterpret_cast(_gst_332) + (0)) = dg_o_22; } if (t_22 < clen) { { - auto _gst_301 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_301)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_333 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_333)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_302 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_302)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_302) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_334 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_334)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_23 = 16 + 8 * th + 7; @@ -4071,24 +4305,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; - float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23) + dgf_23; + float qe_22 = q_t0_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22)) + dgf_23; int goff_23 = t_23 * rs_g; int qoff_23 = t_23 * qrs; { - auto _gst_303 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_303)); - *(reinterpret_cast(_gst_303) + (0)) = dg_o_23; + auto _gst_335 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_335)); + *(reinterpret_cast(_gst_335) + (0)) = dg_o_23; } if (t_23 < clen) { { - auto _gst_304 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_304)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_336 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_336)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_336) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_305 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_305)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_305) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_337 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_337)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dk_o_23; } } int b4_1 = lane_1 >> 4 & 1; @@ -4096,31 +4335,31 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_306; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_338; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_307; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_339; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_308; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_340; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_309; + float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_341; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_310; + float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_342; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_311; + float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_343; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_312; - float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_313; - float _shfl_xor_sync_314 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_314; + float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_344; + float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_345; + float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_346; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); mbarrier_wait(epi_full_addr + (th) * 8, 0); @@ -4173,24 +4412,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; - float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24) + dgf_24; + float qe_23 = q_t0_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23)) + dgf_24; int goff_24 = t_24 * rs_g; int qoff_24 = t_24 * qrs; { - auto _gst_315 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_o_24; + auto _gst_347 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_o_24; } if (t_24 < clen) { { - auto _gst_316 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_348 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_348) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_317 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_349 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_349) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_25 = 8 * th + 1; @@ -4216,24 +4460,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_25 = dk2_25 * bt_t_25; float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; - float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25) + dgf_25; + float qe_24 = q_t0_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24)) + dgf_25; int goff_25 = t_25 * rs_g; int qoff_25 = t_25 * qrs; { - auto _gst_318 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = dg_o_25; + auto _gst_350 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_350)); + *(reinterpret_cast(_gst_350) + (0)) = dg_o_25; } if (t_25 < clen) { { - auto _gst_319 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_351 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_320 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_352 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_26 = 8 * th + 2; @@ -4259,24 +4508,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_26 = dk2_26 * bt_t_26; float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; - float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26) + dgf_26; + float qe_25 = q_t0_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25)) + dgf_26; int goff_26 = t_26 * rs_g; int qoff_26 = t_26 * qrs; { - auto _gst_321 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast(_gst_321) + (0)) = dg_o_26; + auto _gst_353 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast(_gst_353) + (0)) = dg_o_26; } if (t_26 < clen) { { - auto _gst_322 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_354 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_323 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_355 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_27 = 8 * th + 3; @@ -4302,24 +4556,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_27 = dk2_27 * bt_t_27; float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; - float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27) + dgf_27; + float qe_26 = q_t0_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26)) + dgf_27; int goff_27 = t_27 * rs_g; int qoff_27 = t_27 * qrs; { - auto _gst_324 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast(_gst_324) + (0)) = dg_o_27; + auto _gst_356 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast(_gst_356) + (0)) = dg_o_27; } if (t_27 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_28 = 8 * th + 4; @@ -4345,24 +4604,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_28 = dk2_28 * bt_t_28; float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; - float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28) + dgf_28; + float qe_27 = q_t0_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27)) + dgf_28; int goff_28 = t_28 * rs_g; int qoff_28 = t_28 * qrs; { - auto _gst_327 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast(_gst_327) + (0)) = dg_o_28; + auto _gst_359 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast(_gst_359) + (0)) = dg_o_28; } if (t_28 < clen) { { - auto _gst_328 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_360 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_329 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_361 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_29 = 8 * th + 5; @@ -4388,24 +4652,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_29 = dk2_29 * bt_t_29; float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; - float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29) + dgf_29; + float qe_28 = q_t0_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28)) + dgf_29; int goff_29 = t_29 * rs_g; int qoff_29 = t_29 * qrs; { - auto _gst_330 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast(_gst_330) + (0)) = dg_o_29; + auto _gst_362 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast(_gst_362) + (0)) = dg_o_29; } if (t_29 < clen) { { - auto _gst_331 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_363 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_332 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_364 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_30 = 8 * th + 6; @@ -4431,24 +4700,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_30 = dk2_30 * bt_t_30; float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; - float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30) + dgf_30; + float qe_29 = q_t0_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29)) + dgf_30; int goff_30 = t_30 * rs_g; int qoff_30 = t_30 * qrs; { - auto _gst_333 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast(_gst_333) + (0)) = dg_o_30; + auto _gst_365 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast(_gst_365) + (0)) = dg_o_30; } if (t_30 < clen) { { - auto _gst_334 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_366 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_335 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_335)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_335) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_367 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_367)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_367) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_31 = 8 * th + 7; @@ -4474,24 +4748,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_31 = dk2_31 * bt_t_31; float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; - float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31) + dgf_31; + float qe_30 = q_t0_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30)) + dgf_31; int goff_31 = t_31 * rs_g; int qoff_31 = t_31 * qrs; { - auto _gst_336 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_336)); - *(reinterpret_cast(_gst_336) + (0)) = dg_o_31; + auto _gst_368 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_368)); + *(reinterpret_cast(_gst_368) + (0)) = dg_o_31; } if (t_31 < clen) { { - auto _gst_337 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_337)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_369 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_369)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_369) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_338 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_338)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_338) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_370 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_370)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_370) + (0)) = (__nv_bfloat16)dk_o_31; } } int b4_2 = lane_1 >> 4 & 1; @@ -4499,31 +4778,31 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_371; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_372; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_341; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_373; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_342; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_374; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_343; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_375; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_344; + float _shfl_xor_sync_376 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_376; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_345; - float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_346; - float _shfl_xor_sync_347 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_347; + float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_377; + float _shfl_xor_sync_378 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_378; + float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_379; if ((lane_1 & 3) == 0) { int e_star = lane_1 >> 2 & 7; pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_2; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu index c24eedc43de..7a39474948b 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -200,6 +201,21 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { } +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + __device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { float r; asm("mul.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); @@ -2281,12 +2297,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_0); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_0 : __fmaf_rn(dl, _rcp_170, e2_0)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2310,12 +2332,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_1); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_1, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_1, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_1 : __fmaf_rn(dl_1, _rcp_174, e2_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2339,12 +2367,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_2); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_2, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_2, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_2 : __fmaf_rn(dl_3, _rcp_178, e2_2)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2368,12 +2402,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_3); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_3, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_3, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_3 : __fmaf_rn(dl_5, _rcp_182, e2_3)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2409,12 +2449,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2434,12 +2480,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2459,12 +2511,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2484,12 +2542,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2537,12 +2601,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2566,12 +2636,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_1 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2595,12 +2671,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2624,12 +2706,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2671,12 +2759,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2700,12 +2794,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2729,12 +2829,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2758,12 +2864,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2834,37 +2946,37 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 d2_56 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_56.x); float e1_12 = approx_exp2(d2_56.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_57.x); float e1_13 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_58.x); float e1_14 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_59.x); float e1_15 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2876,29 +2988,29 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2906,29 +3018,29 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2998,7 +3110,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3030,7 +3147,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3062,7 +3184,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_1)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3094,7 +3221,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_2)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3126,7 +3258,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_3)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3158,7 +3295,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_4)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3190,7 +3332,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_5)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3222,7 +3369,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_6)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3237,31 +3389,31 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_216 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_216; + float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_248; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_217 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_217; + float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_249; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_218 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_218; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_250; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_219 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_219; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_251; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_220 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_220; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_252; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_221 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_221; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_253; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_222 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_222; - float _shfl_xor_sync_223 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_223; - float _shfl_xor_sync_224 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_224; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_254; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_255; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_256; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); int ew_0 = 98304 - 32768 * th >> 2; @@ -3271,8 +3423,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM for (int i = 0; i < 2; i++) { int j = qd + 4 * i; bool live = 32 + 8 * th + j < clen; - int _min_225 = ((32 + 8 * th + j) < (clen - 1) ? (32 + 8 * th + j) : (clen - 1)); - long long row = ext0 + (long long)_min_225; + int _min_257 = ((32 + 8 * th + j) < (clen - 1) ? (32 + 8 * th + j) : (clen - 1)); + long long row = ext0 + (long long)_min_257; float gq[4]; float gk_0[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" @@ -3289,26 +3441,26 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float rq[1]; float rk[1]; { - uint2 _vld_226; - _vld_226 = *reinterpret_cast(q_raw + (row * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_226 = reinterpret_cast(&_vld_226); + uint2 _vld_258; + _vld_258 = *reinterpret_cast(q_raw + (row * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_258 = reinterpret_cast(&_vld_258); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xq[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_226[_pair]) << 16); - (&xq[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_226[_pair]) & 0xffff0000u); + (&xq[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_258[_pair]) << 16); + (&xq[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_258[_pair]) & 0xffff0000u); } } { rq[0] = *reinterpret_cast(qk_rstd + ((0 + row) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_227; - _vld_227 = *reinterpret_cast(k_raw + (row * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_227 = reinterpret_cast(&_vld_227); + uint2 _vld_259; + _vld_259 = *reinterpret_cast(k_raw + (row * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_259 = reinterpret_cast(&_vld_259); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xk[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_227[_pair]) << 16); - (&xk[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_227[_pair]) & 0xffff0000u); + (&xk[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_259[_pair]) << 16); + (&xk[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_259[_pair]) & 0xffff0000u); } } { @@ -3318,106 +3470,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y[4]; float p[4]; y[0] = mul_f32_rn_noftz(xq[0], rq[0]); - float _fma_228; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_228) : "f"(gq[0]), "f"(y[0]), "f"(nz)); - p[0] = _fma_228; + float _fma_260; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_260) : "f"(gq[0]), "f"(y[0]), "f"(nz)); + p[0] = _fma_260; y[1] = mul_f32_rn_noftz(xq[1], rq[0]); - float _fma_229; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_229) : "f"(gq[1]), "f"(y[1]), "f"(nz)); - p[1] = _fma_229; + float _fma_261; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_261) : "f"(gq[1]), "f"(y[1]), "f"(nz)); + p[1] = _fma_261; y[2] = mul_f32_rn_noftz(xq[2], rq[0]); - float _fma_230; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_230) : "f"(gq[2]), "f"(y[2]), "f"(nz)); - p[2] = _fma_230; + float _fma_262; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_262) : "f"(gq[2]), "f"(y[2]), "f"(nz)); + p[2] = _fma_262; y[3] = mul_f32_rn_noftz(xq[3], rq[0]); - float _fma_231; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_231) : "f"(gq[3]), "f"(y[3]), "f"(nz)); - p[3] = _fma_231; - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, p[0], 4); - float _add_232; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_232) : "f"(p[0]), "f"(_shfl_xor_sync_233)); - p[0] = _add_232; - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, p[1], 4); - float _add_234; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_234) : "f"(p[1]), "f"(_shfl_xor_sync_235)); - p[1] = _add_234; - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, p[2], 4); - float _add_236; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_236) : "f"(p[2]), "f"(_shfl_xor_sync_237)); - p[2] = _add_236; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, p[3], 4); - float _add_238; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_238) : "f"(p[3]), "f"(_shfl_xor_sync_239)); - p[3] = _add_238; - float _shfl_xor_sync_241 = __shfl_xor_sync(0xFFFFFFFF, p[0], 2); - float _add_240; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_240) : "f"(p[0]), "f"(_shfl_xor_sync_241)); - p[0] = _add_240; - float _shfl_xor_sync_243 = __shfl_xor_sync(0xFFFFFFFF, p[1], 2); - float _add_242; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_242) : "f"(p[1]), "f"(_shfl_xor_sync_243)); - p[1] = _add_242; - float _shfl_xor_sync_245 = __shfl_xor_sync(0xFFFFFFFF, p[2], 2); - float _add_244; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_244) : "f"(p[2]), "f"(_shfl_xor_sync_245)); - p[2] = _add_244; - float _shfl_xor_sync_247 = __shfl_xor_sync(0xFFFFFFFF, p[3], 2); - float _add_246; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_246) : "f"(p[3]), "f"(_shfl_xor_sync_247)); - p[3] = _add_246; - float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, p[0], 1); - float _add_248; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_248) : "f"(p[0]), "f"(_shfl_xor_sync_249)); - p[0] = _add_248; - float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, p[1], 1); - float _add_250; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_250) : "f"(p[1]), "f"(_shfl_xor_sync_251)); - p[1] = _add_250; - float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, p[2], 1); - float _add_252; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_252) : "f"(p[2]), "f"(_shfl_xor_sync_253)); - p[2] = _add_252; - float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, p[3], 1); - float _add_254; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_254) : "f"(p[3]), "f"(_shfl_xor_sync_255)); - p[3] = _add_254; - float _add_256; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_256) : "f"(p[0]), "f"(p[2])); - float _add_257; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_257) : "f"(p[1]), "f"(p[3])); + float _fma_263; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_263) : "f"(gq[3]), "f"(y[3]), "f"(nz)); + p[3] = _fma_263; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, p[0], 4); + float _add_264; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_264) : "f"(p[0]), "f"(_shfl_xor_sync_265)); + p[0] = _add_264; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, p[1], 4); + float _add_266; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_266) : "f"(p[1]), "f"(_shfl_xor_sync_267)); + p[1] = _add_266; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, p[2], 4); + float _add_268; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_268) : "f"(p[2]), "f"(_shfl_xor_sync_269)); + p[2] = _add_268; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, p[3], 4); + float _add_270; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_270) : "f"(p[3]), "f"(_shfl_xor_sync_271)); + p[3] = _add_270; + float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, p[0], 2); + float _add_272; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_272) : "f"(p[0]), "f"(_shfl_xor_sync_273)); + p[0] = _add_272; + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, p[1], 2); + float _add_274; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_274) : "f"(p[1]), "f"(_shfl_xor_sync_275)); + p[1] = _add_274; + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, p[2], 2); + float _add_276; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_276) : "f"(p[2]), "f"(_shfl_xor_sync_277)); + p[2] = _add_276; + float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, p[3], 2); + float _add_278; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_278) : "f"(p[3]), "f"(_shfl_xor_sync_279)); + p[3] = _add_278; + float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, p[0], 1); + float _add_280; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_280) : "f"(p[0]), "f"(_shfl_xor_sync_281)); + p[0] = _add_280; + float _shfl_xor_sync_283 = __shfl_xor_sync(0xFFFFFFFF, p[1], 1); + float _add_282; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_282) : "f"(p[1]), "f"(_shfl_xor_sync_283)); + p[1] = _add_282; + float _shfl_xor_sync_285 = __shfl_xor_sync(0xFFFFFFFF, p[2], 1); + float _add_284; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_284) : "f"(p[2]), "f"(_shfl_xor_sync_285)); + p[2] = _add_284; + float _shfl_xor_sync_287 = __shfl_xor_sync(0xFFFFFFFF, p[3], 1); + float _add_286; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_286) : "f"(p[3]), "f"(_shfl_xor_sync_287)); + p[3] = _add_286; + float _add_288; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_288) : "f"(p[0]), "f"(p[2])); + float _add_289; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_289) : "f"(p[1]), "f"(p[3])); float sq; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq) : "f"(_add_256), "f"(_add_257)); - float _shfl_sync_260 = __shfl_sync(0xFFFFFFFF, sq, 0); - float _shfl_sync_261 = __shfl_sync(0xFFFFFFFF, sq, 8); - float _add_259; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_259) : "f"(_shfl_sync_260), "f"(_shfl_sync_261)); - float _shfl_sync_262 = __shfl_sync(0xFFFFFFFF, sq, 16); - float _add_258; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_258) : "f"(_add_259), "f"(_shfl_sync_262)); - float _shfl_sync_263 = __shfl_sync(0xFFFFFFFF, sq, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq) : "f"(_add_288), "f"(_add_289)); + float _shfl_sync_292 = __shfl_sync(0xFFFFFFFF, sq, 0); + float _shfl_sync_293 = __shfl_sync(0xFFFFFFFF, sq, 8); + float _add_291; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_291) : "f"(_shfl_sync_292), "f"(_shfl_sync_293)); + float _shfl_sync_294 = __shfl_sync(0xFFFFFFFF, sq, 16); + float _add_290; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_290) : "f"(_add_291), "f"(_shfl_sync_294)); + float _shfl_sync_295 = __shfl_sync(0xFFFFFFFF, sq, 24); float dot; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_258), "f"(_shfl_sync_263)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_290), "f"(_shfl_sync_295)); float o[4]; - float _fma_265; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_265) : "f"(y[0]), "f"(dot), "f"(nz)); - float _sub_264; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_264) : "f"(gq[0]), "f"(_fma_265)); - o[0] = mul_f32_rn_noftz(rq[0], _sub_264); - float _fma_267; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_267) : "f"(y[1]), "f"(dot), "f"(nz)); - float _sub_266; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_266) : "f"(gq[1]), "f"(_fma_267)); - o[1] = mul_f32_rn_noftz(rq[0], _sub_266); - float _fma_269; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_269) : "f"(y[2]), "f"(dot), "f"(nz)); - float _sub_268; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_268) : "f"(gq[2]), "f"(_fma_269)); - o[2] = mul_f32_rn_noftz(rq[0], _sub_268); - float _fma_271; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_271) : "f"(y[3]), "f"(dot), "f"(nz)); - float _sub_270; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_270) : "f"(gq[3]), "f"(_fma_271)); - o[3] = mul_f32_rn_noftz(rq[0], _sub_270); + float _fma_297; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_297) : "f"(y[0]), "f"(dot), "f"(nz)); + float _sub_296; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_296) : "f"(gq[0]), "f"(_fma_297)); + o[0] = mul_f32_rn_noftz(rq[0], _sub_296); + float _fma_299; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_299) : "f"(y[1]), "f"(dot), "f"(nz)); + float _sub_298; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_298) : "f"(gq[1]), "f"(_fma_299)); + o[1] = mul_f32_rn_noftz(rq[0], _sub_298); + float _fma_301; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_301) : "f"(y[2]), "f"(dot), "f"(nz)); + float _sub_300; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_300) : "f"(gq[2]), "f"(_fma_301)); + o[2] = mul_f32_rn_noftz(rq[0], _sub_300); + float _fma_303; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_303) : "f"(y[3]), "f"(dot), "f"(nz)); + float _sub_302; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_302) : "f"(gq[3]), "f"(_fma_303)); + o[3] = mul_f32_rn_noftz(rq[0], _sub_302); unsigned int ob[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -3428,106 +3580,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_0[4]; float p_0[4]; y_0[0] = mul_f32_rn_noftz(xk[0], rk[0]); - float _fma_272; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_272) : "f"(gk_0[0]), "f"(y_0[0]), "f"(nz)); - p_0[0] = _fma_272; + float _fma_304; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_304) : "f"(gk_0[0]), "f"(y_0[0]), "f"(nz)); + p_0[0] = _fma_304; y_0[1] = mul_f32_rn_noftz(xk[1], rk[0]); - float _fma_273; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_273) : "f"(gk_0[1]), "f"(y_0[1]), "f"(nz)); - p_0[1] = _fma_273; + float _fma_305; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_305) : "f"(gk_0[1]), "f"(y_0[1]), "f"(nz)); + p_0[1] = _fma_305; y_0[2] = mul_f32_rn_noftz(xk[2], rk[0]); - float _fma_274; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_274) : "f"(gk_0[2]), "f"(y_0[2]), "f"(nz)); - p_0[2] = _fma_274; + float _fma_306; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_306) : "f"(gk_0[2]), "f"(y_0[2]), "f"(nz)); + p_0[2] = _fma_306; y_0[3] = mul_f32_rn_noftz(xk[3], rk[0]); - float _fma_275; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_275) : "f"(gk_0[3]), "f"(y_0[3]), "f"(nz)); - p_0[3] = _fma_275; - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 4); - float _add_276; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_276) : "f"(p_0[0]), "f"(_shfl_xor_sync_277)); - p_0[0] = _add_276; - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 4); - float _add_278; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_278) : "f"(p_0[1]), "f"(_shfl_xor_sync_279)); - p_0[1] = _add_278; - float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 4); - float _add_280; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_280) : "f"(p_0[2]), "f"(_shfl_xor_sync_281)); - p_0[2] = _add_280; - float _shfl_xor_sync_283 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 4); - float _add_282; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_282) : "f"(p_0[3]), "f"(_shfl_xor_sync_283)); - p_0[3] = _add_282; - float _shfl_xor_sync_285 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 2); - float _add_284; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_284) : "f"(p_0[0]), "f"(_shfl_xor_sync_285)); - p_0[0] = _add_284; - float _shfl_xor_sync_287 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 2); - float _add_286; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_286) : "f"(p_0[1]), "f"(_shfl_xor_sync_287)); - p_0[1] = _add_286; - float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 2); - float _add_288; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_288) : "f"(p_0[2]), "f"(_shfl_xor_sync_289)); - p_0[2] = _add_288; - float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 2); - float _add_290; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_290) : "f"(p_0[3]), "f"(_shfl_xor_sync_291)); - p_0[3] = _add_290; - float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 1); - float _add_292; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_292) : "f"(p_0[0]), "f"(_shfl_xor_sync_293)); - p_0[0] = _add_292; - float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 1); - float _add_294; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_294) : "f"(p_0[1]), "f"(_shfl_xor_sync_295)); - p_0[1] = _add_294; - float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 1); - float _add_296; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_296) : "f"(p_0[2]), "f"(_shfl_xor_sync_297)); - p_0[2] = _add_296; - float _shfl_xor_sync_299 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 1); - float _add_298; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_298) : "f"(p_0[3]), "f"(_shfl_xor_sync_299)); - p_0[3] = _add_298; - float _add_300; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_300) : "f"(p_0[0]), "f"(p_0[2])); - float _add_301; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_301) : "f"(p_0[1]), "f"(p_0[3])); + float _fma_307; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_307) : "f"(gk_0[3]), "f"(y_0[3]), "f"(nz)); + p_0[3] = _fma_307; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 4); + float _add_308; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_308) : "f"(p_0[0]), "f"(_shfl_xor_sync_309)); + p_0[0] = _add_308; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 4); + float _add_310; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_310) : "f"(p_0[1]), "f"(_shfl_xor_sync_311)); + p_0[1] = _add_310; + float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 4); + float _add_312; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_312) : "f"(p_0[2]), "f"(_shfl_xor_sync_313)); + p_0[2] = _add_312; + float _shfl_xor_sync_315 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 4); + float _add_314; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_314) : "f"(p_0[3]), "f"(_shfl_xor_sync_315)); + p_0[3] = _add_314; + float _shfl_xor_sync_317 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 2); + float _add_316; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_316) : "f"(p_0[0]), "f"(_shfl_xor_sync_317)); + p_0[0] = _add_316; + float _shfl_xor_sync_319 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 2); + float _add_318; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_318) : "f"(p_0[1]), "f"(_shfl_xor_sync_319)); + p_0[1] = _add_318; + float _shfl_xor_sync_321 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 2); + float _add_320; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_320) : "f"(p_0[2]), "f"(_shfl_xor_sync_321)); + p_0[2] = _add_320; + float _shfl_xor_sync_323 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 2); + float _add_322; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_322) : "f"(p_0[3]), "f"(_shfl_xor_sync_323)); + p_0[3] = _add_322; + float _shfl_xor_sync_325 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 1); + float _add_324; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(p_0[0]), "f"(_shfl_xor_sync_325)); + p_0[0] = _add_324; + float _shfl_xor_sync_327 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 1); + float _add_326; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_326) : "f"(p_0[1]), "f"(_shfl_xor_sync_327)); + p_0[1] = _add_326; + float _shfl_xor_sync_329 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 1); + float _add_328; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(p_0[2]), "f"(_shfl_xor_sync_329)); + p_0[2] = _add_328; + float _shfl_xor_sync_331 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 1); + float _add_330; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(p_0[3]), "f"(_shfl_xor_sync_331)); + p_0[3] = _add_330; + float _add_332; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_332) : "f"(p_0[0]), "f"(p_0[2])); + float _add_333; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_333) : "f"(p_0[1]), "f"(p_0[3])); float sq_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_0) : "f"(_add_300), "f"(_add_301)); - float _shfl_sync_304 = __shfl_sync(0xFFFFFFFF, sq_0, 0); - float _shfl_sync_305 = __shfl_sync(0xFFFFFFFF, sq_0, 8); - float _add_303; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_303) : "f"(_shfl_sync_304), "f"(_shfl_sync_305)); - float _shfl_sync_306 = __shfl_sync(0xFFFFFFFF, sq_0, 16); - float _add_302; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_302) : "f"(_add_303), "f"(_shfl_sync_306)); - float _shfl_sync_307 = __shfl_sync(0xFFFFFFFF, sq_0, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_0) : "f"(_add_332), "f"(_add_333)); + float _shfl_sync_336 = __shfl_sync(0xFFFFFFFF, sq_0, 0); + float _shfl_sync_337 = __shfl_sync(0xFFFFFFFF, sq_0, 8); + float _add_335; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_335) : "f"(_shfl_sync_336), "f"(_shfl_sync_337)); + float _shfl_sync_338 = __shfl_sync(0xFFFFFFFF, sq_0, 16); + float _add_334; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(_add_335), "f"(_shfl_sync_338)); + float _shfl_sync_339 = __shfl_sync(0xFFFFFFFF, sq_0, 24); float dot_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_302), "f"(_shfl_sync_307)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_334), "f"(_shfl_sync_339)); float o_0[4]; - float _fma_309; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_309) : "f"(y_0[0]), "f"(dot_0), "f"(nz)); - float _sub_308; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_308) : "f"(gk_0[0]), "f"(_fma_309)); - o_0[0] = mul_f32_rn_noftz(rk[0], _sub_308); - float _fma_311; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_311) : "f"(y_0[1]), "f"(dot_0), "f"(nz)); - float _sub_310; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_310) : "f"(gk_0[1]), "f"(_fma_311)); - o_0[1] = mul_f32_rn_noftz(rk[0], _sub_310); - float _fma_313; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_313) : "f"(y_0[2]), "f"(dot_0), "f"(nz)); - float _sub_312; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_312) : "f"(gk_0[2]), "f"(_fma_313)); - o_0[2] = mul_f32_rn_noftz(rk[0], _sub_312); - float _fma_315; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_315) : "f"(y_0[3]), "f"(dot_0), "f"(nz)); - float _sub_314; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_314) : "f"(gk_0[3]), "f"(_fma_315)); - o_0[3] = mul_f32_rn_noftz(rk[0], _sub_314); + float _fma_341; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_341) : "f"(y_0[0]), "f"(dot_0), "f"(nz)); + float _sub_340; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_340) : "f"(gk_0[0]), "f"(_fma_341)); + o_0[0] = mul_f32_rn_noftz(rk[0], _sub_340); + float _fma_343; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_343) : "f"(y_0[1]), "f"(dot_0), "f"(nz)); + float _sub_342; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_342) : "f"(gk_0[1]), "f"(_fma_343)); + o_0[1] = mul_f32_rn_noftz(rk[0], _sub_342); + float _fma_345; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_345) : "f"(y_0[2]), "f"(dot_0), "f"(nz)); + float _sub_344; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_344) : "f"(gk_0[2]), "f"(_fma_345)); + o_0[2] = mul_f32_rn_noftz(rk[0], _sub_344); + float _fma_347; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_347) : "f"(y_0[3]), "f"(dot_0), "f"(nz)); + float _sub_346; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_346) : "f"(gk_0[3]), "f"(_fma_347)); + o_0[3] = mul_f32_rn_noftz(rk[0], _sub_346); unsigned int ob_0[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -3545,340 +3697,153 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float r_272[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1049 = r_272[0]; - float gc_1050 = gn_1049 - 21.640425f; - int eb_1051 = 98304 + th * -32768; - int ew_1052 = eb_1051 >> 2; - int qbase_1053 = pool_f32_addr + (unsigned int)(eb_1051 + 12288 + dh_lane * 1024); - int sb8_1054 = 48 + 8 * th; - int gb8_1055 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1054 * 128); - int kb8_1056 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1054 * 128); - float dbp_1057[8]; - float _tmem_load_12[8]; - tmem_ld_x8(&_tmem_load_12[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float _tmem_load_13[8]; - tmem_ld_x8(&_tmem_load_13[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float _tmem_load_14[8]; - tmem_ld_x8(&_tmem_load_14[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float _tmem_load_15[8]; - tmem_ld_x8(&_tmem_load_15[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float _tmem_load_16[8]; - tmem_ld_x8(&_tmem_load_16[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_1058[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[0])), "=r"(*reinterpret_cast(&btv_1058[(0) + 1])), "=r"(*reinterpret_cast(&btv_1058[(0) + 2])), "=r"(*reinterpret_cast(&btv_1058[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[4])), "=r"(*reinterpret_cast(&btv_1058[(4) + 1])), "=r"(*reinterpret_cast(&btv_1058[(4) + 2])), "=r"(*reinterpret_cast(&btv_1058[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); + float gn_0 = r_272[0]; + float gc_0 = gn_0 - 21.640425f; + int eb_0 = 98304 + th * -32768; + int ew_1 = eb_0 >> 2; + int qbase_0 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); + int sb8_0 = 48 + 8 * th; + int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); + int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); + int c_qa = 160 + 8 * th; + int c_ka = 176 + 8 * th; + int c_qd = 224 + 8 * th; + int c_kd = 240 + 8 * th; + int c_dk = 48 + 8 * th; + int c_bp = c_ka; + float s_run_0[1]; + s_run_0[0] = 0.0f; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j_1 = 7 - k; + float a_qd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j_1))); + float a_kd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd + j_1))); + float a_dk[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk + j_1))); + float a_qa[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa + j_1))); + float a_ka[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka + j_1))); + float btr[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_0 + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8_0 + j_1; + float r_273[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_273[0]; + unsigned int r_274[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_274[0] << 16); + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_275[0] << 16); + float q_m = ((t >= clen) ? 0.0f : q_t); + float dqf = pool_f32[ew_1 + 128 * j_1 + d_lane]; + float dkf = pool_f32[ew_1 + (1024 + 128 * j_1) + d_lane]; + float dgf = pool_f32[ew_1 + (2048 + 128 * j_1) + d_lane]; + float bt_t = btr[0]; + float e_dg = approx_exp2(g_t - gc_0); + float e_off = approx_exp2(g_t - gn_0); + float dq2 = __fmaf_rn(a_qa[0], e_off, a_qd[0] * e_dg); + float dk2 = __fmaf_rn(a_ka[0], e_off, a_kd[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + a_dk[0]; + float qe_7 = q_m * e_dg; + float kp_19 = k_t * bt_t; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, a_qd[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg) * a_kd[0]); + float dg_o = __fmaf_rn(q_m, dq2, __fmaf_rn(dk2b - a_dk[0], k_t, cr_7)) + dgf; + s_run_0[0] = s_run_0[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j_1)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j_1)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(98304 - 32768 * th >> 2) + (128 * j_1 + d_lane)] = dq_o; + pool_f32[(98304 - 32768 * th >> 2) + (1024 + 128 * j_1 + d_lane)] = dk_o; + } + float tot_0 = s_run_0[0]; + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1059[1]; - s_run_1059[0] = 0.0f; - int t_0 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew_1052 + 896 + d_lane]; - float dkf_0 = pool_f32[ew_1052 + 1920 + d_lane]; - float dgf_0 = pool_f32[ew_1052 + 2944 + d_lane]; - float bt_t_0 = btv_1058[7]; - float e_off_0 = approx_exp2(g_t_0 - gn_1049); - float e_dg_0 = approx_exp2(g_t_0 - gc_1050); - float dq2_0 = __fmaf_rn(_tmem_load_12[7], e_off_0, _tmem_load_14[7] * e_dg_0); - float dk2_0 = __fmaf_rn(_tmem_load_13[7], e_off_0, _tmem_load_15[7] * e_dg_0); - dbp_1057[7] = dk2_0 * k_t_0; - float dk2b_0 = dk2_0 * bt_t_0; - float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; - s_run_1059[0] = s_run_1059[0] + dg_o_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_0; - pool_f32[(98304 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_0; - int t_1 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew_1052 + 768 + d_lane]; - float dkf_1 = pool_f32[ew_1052 + 1792 + d_lane]; - float dgf_1 = pool_f32[ew_1052 + 2816 + d_lane]; - float bt_t_1 = btv_1058[6]; - float e_off_1 = approx_exp2(g_t_1 - gn_1049); - float e_dg_1 = approx_exp2(g_t_1 - gc_1050); - float dq2_1 = __fmaf_rn(_tmem_load_12[6], e_off_1, _tmem_load_14[6] * e_dg_1); - float dk2_1 = __fmaf_rn(_tmem_load_13[6], e_off_1, _tmem_load_15[6] * e_dg_1); - dbp_1057[6] = dk2_1 * k_t_1; - float dk2b_1 = dk2_1 * bt_t_1; - float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + _tmem_load_16[6]; - float dg_o_1 = __fmaf_rn(q_t_1_9, dq2_1, (dk2b_1 - _tmem_load_16[6]) * k_t_1) + dgf_1; - s_run_1059[0] = s_run_1059[0] + dg_o_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_1; - pool_f32[(98304 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_1; - int t_2 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew_1052 + 640 + d_lane]; - float dkf_2 = pool_f32[ew_1052 + 1664 + d_lane]; - float dgf_2 = pool_f32[ew_1052 + 2688 + d_lane]; - float bt_t_2 = btv_1058[5]; - float e_off_2 = approx_exp2(g_t_2 - gn_1049); - float e_dg_2 = approx_exp2(g_t_2 - gc_1050); - float dq2_2 = __fmaf_rn(_tmem_load_12[5], e_off_2, _tmem_load_14[5] * e_dg_2); - float dk2_2 = __fmaf_rn(_tmem_load_13[5], e_off_2, _tmem_load_15[5] * e_dg_2); - dbp_1057[5] = dk2_2 * k_t_2; - float dk2b_2 = dk2_2 * bt_t_2; - float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[5]; - float dg_o_2 = __fmaf_rn(q_t_1_10, dq2_2, (dk2b_2 - _tmem_load_16[5]) * k_t_2) + dgf_2; - s_run_1059[0] = s_run_1059[0] + dg_o_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_2; - pool_f32[(98304 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_2; - int t_3 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew_1052 + 512 + d_lane]; - float dkf_3 = pool_f32[ew_1052 + 1536 + d_lane]; - float dgf_3 = pool_f32[ew_1052 + 2560 + d_lane]; - float bt_t_3 = btv_1058[4]; - float e_off_3 = approx_exp2(g_t_3 - gn_1049); - float e_dg_3 = approx_exp2(g_t_3 - gc_1050); - float dq2_3 = __fmaf_rn(_tmem_load_12[4], e_off_3, _tmem_load_14[4] * e_dg_3); - float dk2_3 = __fmaf_rn(_tmem_load_13[4], e_off_3, _tmem_load_15[4] * e_dg_3); - dbp_1057[4] = dk2_3 * k_t_3; - float dk2b_3 = dk2_3 * bt_t_3; - float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[4]; - float dg_o_3 = __fmaf_rn(q_t_1_11, dq2_3, (dk2b_3 - _tmem_load_16[4]) * k_t_3) + dgf_3; - s_run_1059[0] = s_run_1059[0] + dg_o_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_3; - pool_f32[(98304 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_3; - int t_4 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew_1052 + 384 + d_lane]; - float dkf_4 = pool_f32[ew_1052 + 1408 + d_lane]; - float dgf_4 = pool_f32[ew_1052 + 2432 + d_lane]; - float bt_t_4 = btv_1058[3]; - float e_off_4 = approx_exp2(g_t_4 - gn_1049); - float e_dg_4 = approx_exp2(g_t_4 - gc_1050); - float dq2_4 = __fmaf_rn(_tmem_load_12[3], e_off_4, _tmem_load_14[3] * e_dg_4); - float dk2_4 = __fmaf_rn(_tmem_load_13[3], e_off_4, _tmem_load_15[3] * e_dg_4); - dbp_1057[3] = dk2_4 * k_t_4; - float dk2b_4 = dk2_4 * bt_t_4; - float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[3]; - float dg_o_4 = __fmaf_rn(q_t_1_12, dq2_4, (dk2b_4 - _tmem_load_16[3]) * k_t_4) + dgf_4; - s_run_1059[0] = s_run_1059[0] + dg_o_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_4; - pool_f32[(98304 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_4; - int t_5 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew_1052 + 256 + d_lane]; - float dkf_5 = pool_f32[ew_1052 + 1280 + d_lane]; - float dgf_5 = pool_f32[ew_1052 + 2304 + d_lane]; - float bt_t_5 = btv_1058[2]; - float e_off_5 = approx_exp2(g_t_5 - gn_1049); - float e_dg_5 = approx_exp2(g_t_5 - gc_1050); - float dq2_5 = __fmaf_rn(_tmem_load_12[2], e_off_5, _tmem_load_14[2] * e_dg_5); - float dk2_5 = __fmaf_rn(_tmem_load_13[2], e_off_5, _tmem_load_15[2] * e_dg_5); - dbp_1057[2] = dk2_5 * k_t_5; - float dk2b_5 = dk2_5 * bt_t_5; - float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[2]; - float dg_o_5 = __fmaf_rn(q_t_1_13, dq2_5, (dk2b_5 - _tmem_load_16[2]) * k_t_5) + dgf_5; - s_run_1059[0] = s_run_1059[0] + dg_o_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_5; - pool_f32[(98304 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_5; - int t_6 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew_1052 + 128 + d_lane]; - float dkf_6 = pool_f32[ew_1052 + 1152 + d_lane]; - float dgf_6 = pool_f32[ew_1052 + 2176 + d_lane]; - float bt_t_6 = btv_1058[1]; - float e_off_6 = approx_exp2(g_t_6 - gn_1049); - float e_dg_6 = approx_exp2(g_t_6 - gc_1050); - float dq2_6 = __fmaf_rn(_tmem_load_12[1], e_off_6, _tmem_load_14[1] * e_dg_6); - float dk2_6 = __fmaf_rn(_tmem_load_13[1], e_off_6, _tmem_load_15[1] * e_dg_6); - dbp_1057[1] = dk2_6 * k_t_6; - float dk2b_6 = dk2_6 * bt_t_6; - float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[1]; - float dg_o_6 = __fmaf_rn(q_t_1_14, dq2_6, (dk2b_6 - _tmem_load_16[1]) * k_t_6) + dgf_6; - s_run_1059[0] = s_run_1059[0] + dg_o_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_6; - pool_f32[(98304 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_6; - int t_7 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew_1052 + d_lane]; - float dkf_7 = pool_f32[ew_1052 + 1024 + d_lane]; - float dgf_7 = pool_f32[ew_1052 + 2048 + d_lane]; - float bt_t_7 = btv_1058[0]; - float e_off_7 = approx_exp2(g_t_7 - gn_1049); - float e_dg_7 = approx_exp2(g_t_7 - gc_1050); - float dq2_7 = __fmaf_rn(_tmem_load_12[0], e_off_7, _tmem_load_14[0] * e_dg_7); - float dk2_7 = __fmaf_rn(_tmem_load_13[0], e_off_7, _tmem_load_15[0] * e_dg_7); - dbp_1057[0] = dk2_7 * k_t_7; - float dk2b_7 = dk2_7 * bt_t_7; - float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[0]; - float dg_o_7 = __fmaf_rn(q_t_1_15, dq2_7, (dk2b_7 - _tmem_load_16[0]) * k_t_7) + dgf_7; - s_run_1059[0] = s_run_1059[0] + dg_o_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + d_lane] = dq_o_7; - pool_f32[(98304 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_7; - float tot_1228 = s_run_1059[0]; - int b4_1229 = lane_1 >> 4 & 1; - int b3_1230 = lane_1 >> 3 & 1; - int b2_1231 = lane_1 >> 2 & 1; - float v4_1232[4]; - float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_316 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_316; - float send_1 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_317 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_317; - float send_2 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_318 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_318; - float send_3 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_319 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_319; - float v2_1237[2]; - float send_4 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_320 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_320; - float send_5 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_321 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_321; - float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_322 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_322; - float _shfl_xor_sync_323 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_323; - float _shfl_xor_sync_324 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_324; + int b4_0 = lane_1 >> 4 & 1; + int b3_0 = lane_1 >> 3 & 1; + int b2_0 = lane_1 >> 2 & 1; + float v4_0[4]; + float send_0 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); + float _shfl_xor_sync_348 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_348; + float send_1 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); + float _shfl_xor_sync_349 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_349; + float send_2 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); + float _shfl_xor_sync_350 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_350; + float send_3 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); + float _shfl_xor_sync_351 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_351; + float v2[2]; + float send2_0 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); + float _shfl_xor_sync_352 = __shfl_xor_sync(0xFFFFFFFF, send2_0, 8); + v2[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_352; + float send2_1 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); + float _shfl_xor_sync_353 = __shfl_xor_sync(0xFFFFFFFF, send2_1, 8); + v2[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_353; + float send1 = ((b2_0 == 0) ? v2[1] : v2[0]); + float _shfl_xor_sync_354 = __shfl_xor_sync(0xFFFFFFFF, send1, 4); + float v1_0 = ((b2_0 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_354; + float _shfl_xor_sync_355 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_355; + float _shfl_xor_sync_356 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_356; asm volatile("tcgen05.fence::before_thread_sync;"); - int ew_1 = 98304 - 32768 * th >> 2; + int ew_2 = 98304 - 32768 * th >> 2; float nz_0 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); asm volatile("barrier.sync 1, 256;" ::: "memory"); #pragma unroll 1 for (int i_1 = 0; i_1 < 2; i_1++) { - int j_1 = qd + 4 * i_1; - bool live_1 = 48 + 8 * th + j_1 < clen; - int _min_325 = ((48 + 8 * th + j_1) < (clen - 1) ? (48 + 8 * th + j_1) : (clen - 1)); - long long row_1 = ext0 + (long long)_min_325; + int j_2 = qd + 4 * i_1; + bool live_1 = 48 + 8 * th + j_2 < clen; + int _min_357 = ((48 + 8 * th + j_2) < (clen - 1) ? (48 + 8 * th + j_2) : (clen - 1)); + long long row_1 = ext0 + (long long)_min_357; float gq_1[4]; float gk_1[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gq_1[0])), "=r"(*reinterpret_cast(&gq_1[(0) + 1])), "=r"(*reinterpret_cast(&gq_1[(0) + 2])), "=r"(*reinterpret_cast(&gq_1[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_1 + 128 * j_1 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_2 + 128 * j_2 + 4 * lane_1)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gk_1[0])), "=r"(*reinterpret_cast(&gk_1[(0) + 1])), "=r"(*reinterpret_cast(&gk_1[(0) + 2])), "=r"(*reinterpret_cast(&gk_1[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_1 + 1024 + 128 * j_1 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_2 + 1024 + 128 * j_2 + 4 * lane_1)))); if (i_1 == 1) { mbarrier_arrive(epi_empty_addr + (th) * 8); } @@ -3887,26 +3852,26 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float rq_1[1]; float rk_1[1]; { - uint2 _vld_326; - _vld_326 = *reinterpret_cast(q_raw + (row_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_326 = reinterpret_cast(&_vld_326); + uint2 _vld_358; + _vld_358 = *reinterpret_cast(q_raw + (row_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_358 = reinterpret_cast(&_vld_358); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xq_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_326[_pair]) << 16); - (&xq_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_326[_pair]) & 0xffff0000u); + (&xq_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_358[_pair]) << 16); + (&xq_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_358[_pair]) & 0xffff0000u); } } { rq_1[0] = *reinterpret_cast(qk_rstd + ((0 + row_1) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_327; - _vld_327 = *reinterpret_cast(k_raw + (row_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_327 = reinterpret_cast(&_vld_327); + uint2 _vld_359; + _vld_359 = *reinterpret_cast(k_raw + (row_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_359 = reinterpret_cast(&_vld_359); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xk_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_327[_pair]) << 16); - (&xk_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_327[_pair]) & 0xffff0000u); + (&xk_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_359[_pair]) << 16); + (&xk_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_359[_pair]) & 0xffff0000u); } } { @@ -3916,106 +3881,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_1[4]; float p_1[4]; y_1[0] = mul_f32_rn_noftz(xq_1[0], rq_1[0]); - float _fma_328; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_328) : "f"(gq_1[0]), "f"(y_1[0]), "f"(nz_0)); - p_1[0] = _fma_328; + float _fma_360; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_360) : "f"(gq_1[0]), "f"(y_1[0]), "f"(nz_0)); + p_1[0] = _fma_360; y_1[1] = mul_f32_rn_noftz(xq_1[1], rq_1[0]); - float _fma_329; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_329) : "f"(gq_1[1]), "f"(y_1[1]), "f"(nz_0)); - p_1[1] = _fma_329; + float _fma_361; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_361) : "f"(gq_1[1]), "f"(y_1[1]), "f"(nz_0)); + p_1[1] = _fma_361; y_1[2] = mul_f32_rn_noftz(xq_1[2], rq_1[0]); - float _fma_330; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_330) : "f"(gq_1[2]), "f"(y_1[2]), "f"(nz_0)); - p_1[2] = _fma_330; + float _fma_362; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_362) : "f"(gq_1[2]), "f"(y_1[2]), "f"(nz_0)); + p_1[2] = _fma_362; y_1[3] = mul_f32_rn_noftz(xq_1[3], rq_1[0]); - float _fma_331; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_331) : "f"(gq_1[3]), "f"(y_1[3]), "f"(nz_0)); - p_1[3] = _fma_331; - float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 4); - float _add_332; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_332) : "f"(p_1[0]), "f"(_shfl_xor_sync_333)); - p_1[0] = _add_332; - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 4); - float _add_334; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(p_1[1]), "f"(_shfl_xor_sync_335)); - p_1[1] = _add_334; - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 4); - float _add_336; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_336) : "f"(p_1[2]), "f"(_shfl_xor_sync_337)); - p_1[2] = _add_336; - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 4); - float _add_338; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_338) : "f"(p_1[3]), "f"(_shfl_xor_sync_339)); - p_1[3] = _add_338; - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 2); - float _add_340; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_340) : "f"(p_1[0]), "f"(_shfl_xor_sync_341)); - p_1[0] = _add_340; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 2); - float _add_342; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_342) : "f"(p_1[1]), "f"(_shfl_xor_sync_343)); - p_1[1] = _add_342; - float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 2); - float _add_344; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_344) : "f"(p_1[2]), "f"(_shfl_xor_sync_345)); - p_1[2] = _add_344; - float _shfl_xor_sync_347 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 2); - float _add_346; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_346) : "f"(p_1[3]), "f"(_shfl_xor_sync_347)); - p_1[3] = _add_346; - float _shfl_xor_sync_349 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 1); - float _add_348; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_348) : "f"(p_1[0]), "f"(_shfl_xor_sync_349)); - p_1[0] = _add_348; - float _shfl_xor_sync_351 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 1); - float _add_350; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_350) : "f"(p_1[1]), "f"(_shfl_xor_sync_351)); - p_1[1] = _add_350; - float _shfl_xor_sync_353 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 1); - float _add_352; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_352) : "f"(p_1[2]), "f"(_shfl_xor_sync_353)); - p_1[2] = _add_352; - float _shfl_xor_sync_355 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 1); - float _add_354; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_354) : "f"(p_1[3]), "f"(_shfl_xor_sync_355)); - p_1[3] = _add_354; - float _add_356; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_356) : "f"(p_1[0]), "f"(p_1[2])); - float _add_357; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_357) : "f"(p_1[1]), "f"(p_1[3])); + float _fma_363; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_363) : "f"(gq_1[3]), "f"(y_1[3]), "f"(nz_0)); + p_1[3] = _fma_363; + float _shfl_xor_sync_365 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 4); + float _add_364; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_364) : "f"(p_1[0]), "f"(_shfl_xor_sync_365)); + p_1[0] = _add_364; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 4); + float _add_366; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_366) : "f"(p_1[1]), "f"(_shfl_xor_sync_367)); + p_1[1] = _add_366; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 4); + float _add_368; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_368) : "f"(p_1[2]), "f"(_shfl_xor_sync_369)); + p_1[2] = _add_368; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 4); + float _add_370; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_370) : "f"(p_1[3]), "f"(_shfl_xor_sync_371)); + p_1[3] = _add_370; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 2); + float _add_372; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_372) : "f"(p_1[0]), "f"(_shfl_xor_sync_373)); + p_1[0] = _add_372; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 2); + float _add_374; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_374) : "f"(p_1[1]), "f"(_shfl_xor_sync_375)); + p_1[1] = _add_374; + float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 2); + float _add_376; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_376) : "f"(p_1[2]), "f"(_shfl_xor_sync_377)); + p_1[2] = _add_376; + float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 2); + float _add_378; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_378) : "f"(p_1[3]), "f"(_shfl_xor_sync_379)); + p_1[3] = _add_378; + float _shfl_xor_sync_381 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 1); + float _add_380; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_380) : "f"(p_1[0]), "f"(_shfl_xor_sync_381)); + p_1[0] = _add_380; + float _shfl_xor_sync_383 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 1); + float _add_382; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_382) : "f"(p_1[1]), "f"(_shfl_xor_sync_383)); + p_1[1] = _add_382; + float _shfl_xor_sync_385 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 1); + float _add_384; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_384) : "f"(p_1[2]), "f"(_shfl_xor_sync_385)); + p_1[2] = _add_384; + float _shfl_xor_sync_387 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 1); + float _add_386; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_386) : "f"(p_1[3]), "f"(_shfl_xor_sync_387)); + p_1[3] = _add_386; + float _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(p_1[0]), "f"(p_1[2])); + float _add_389; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_389) : "f"(p_1[1]), "f"(p_1[3])); float sq_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1) : "f"(_add_356), "f"(_add_357)); - float _shfl_sync_360 = __shfl_sync(0xFFFFFFFF, sq_1, 0); - float _shfl_sync_361 = __shfl_sync(0xFFFFFFFF, sq_1, 8); - float _add_359; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_359) : "f"(_shfl_sync_360), "f"(_shfl_sync_361)); - float _shfl_sync_362 = __shfl_sync(0xFFFFFFFF, sq_1, 16); - float _add_358; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_358) : "f"(_add_359), "f"(_shfl_sync_362)); - float _shfl_sync_363 = __shfl_sync(0xFFFFFFFF, sq_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1) : "f"(_add_388), "f"(_add_389)); + float _shfl_sync_392 = __shfl_sync(0xFFFFFFFF, sq_1, 0); + float _shfl_sync_393 = __shfl_sync(0xFFFFFFFF, sq_1, 8); + float _add_391; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_391) : "f"(_shfl_sync_392), "f"(_shfl_sync_393)); + float _shfl_sync_394 = __shfl_sync(0xFFFFFFFF, sq_1, 16); + float _add_390; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_390) : "f"(_add_391), "f"(_shfl_sync_394)); + float _shfl_sync_395 = __shfl_sync(0xFFFFFFFF, sq_1, 24); float dot_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_358), "f"(_shfl_sync_363)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_390), "f"(_shfl_sync_395)); float o_1[4]; - float _fma_365; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_365) : "f"(y_1[0]), "f"(dot_1), "f"(nz_0)); - float _sub_364; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_364) : "f"(gq_1[0]), "f"(_fma_365)); - o_1[0] = mul_f32_rn_noftz(rq_1[0], _sub_364); - float _fma_367; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_367) : "f"(y_1[1]), "f"(dot_1), "f"(nz_0)); - float _sub_366; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_366) : "f"(gq_1[1]), "f"(_fma_367)); - o_1[1] = mul_f32_rn_noftz(rq_1[0], _sub_366); - float _fma_369; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_369) : "f"(y_1[2]), "f"(dot_1), "f"(nz_0)); - float _sub_368; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_368) : "f"(gq_1[2]), "f"(_fma_369)); - o_1[2] = mul_f32_rn_noftz(rq_1[0], _sub_368); - float _fma_371; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_371) : "f"(y_1[3]), "f"(dot_1), "f"(nz_0)); - float _sub_370; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_370) : "f"(gq_1[3]), "f"(_fma_371)); - o_1[3] = mul_f32_rn_noftz(rq_1[0], _sub_370); + float _fma_397; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_397) : "f"(y_1[0]), "f"(dot_1), "f"(nz_0)); + float _sub_396; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_396) : "f"(gq_1[0]), "f"(_fma_397)); + o_1[0] = mul_f32_rn_noftz(rq_1[0], _sub_396); + float _fma_399; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_399) : "f"(y_1[1]), "f"(dot_1), "f"(nz_0)); + float _sub_398; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_398) : "f"(gq_1[1]), "f"(_fma_399)); + o_1[1] = mul_f32_rn_noftz(rq_1[0], _sub_398); + float _fma_401; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_401) : "f"(y_1[2]), "f"(dot_1), "f"(nz_0)); + float _sub_400; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_400) : "f"(gq_1[2]), "f"(_fma_401)); + o_1[2] = mul_f32_rn_noftz(rq_1[0], _sub_400); + float _fma_403; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_403) : "f"(y_1[3]), "f"(dot_1), "f"(nz_0)); + float _sub_402; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_402) : "f"(gq_1[3]), "f"(_fma_403)); + o_1[3] = mul_f32_rn_noftz(rq_1[0], _sub_402); unsigned int ob_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -4026,106 +3991,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_1_1[4]; float p_1_1[4]; y_1_1[0] = mul_f32_rn_noftz(xk_1[0], rk_1[0]); - float _fma_372; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_372) : "f"(gk_1[0]), "f"(y_1_1[0]), "f"(nz_0)); - p_1_1[0] = _fma_372; + float _fma_404; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_404) : "f"(gk_1[0]), "f"(y_1_1[0]), "f"(nz_0)); + p_1_1[0] = _fma_404; y_1_1[1] = mul_f32_rn_noftz(xk_1[1], rk_1[0]); - float _fma_373; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_373) : "f"(gk_1[1]), "f"(y_1_1[1]), "f"(nz_0)); - p_1_1[1] = _fma_373; + float _fma_405; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_405) : "f"(gk_1[1]), "f"(y_1_1[1]), "f"(nz_0)); + p_1_1[1] = _fma_405; y_1_1[2] = mul_f32_rn_noftz(xk_1[2], rk_1[0]); - float _fma_374; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_374) : "f"(gk_1[2]), "f"(y_1_1[2]), "f"(nz_0)); - p_1_1[2] = _fma_374; + float _fma_406; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_406) : "f"(gk_1[2]), "f"(y_1_1[2]), "f"(nz_0)); + p_1_1[2] = _fma_406; y_1_1[3] = mul_f32_rn_noftz(xk_1[3], rk_1[0]); - float _fma_375; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_375) : "f"(gk_1[3]), "f"(y_1_1[3]), "f"(nz_0)); - p_1_1[3] = _fma_375; - float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 4); - float _add_376; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_376) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_377)); - p_1_1[0] = _add_376; - float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 4); - float _add_378; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_378) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_379)); - p_1_1[1] = _add_378; - float _shfl_xor_sync_381 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 4); - float _add_380; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_380) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_381)); - p_1_1[2] = _add_380; - float _shfl_xor_sync_383 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 4); - float _add_382; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_382) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_383)); - p_1_1[3] = _add_382; - float _shfl_xor_sync_385 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 2); - float _add_384; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_384) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_385)); - p_1_1[0] = _add_384; - float _shfl_xor_sync_387 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 2); - float _add_386; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_386) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_387)); - p_1_1[1] = _add_386; - float _shfl_xor_sync_389 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 2); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_389)); - p_1_1[2] = _add_388; - float _shfl_xor_sync_391 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 2); - float _add_390; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_390) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_391)); - p_1_1[3] = _add_390; - float _shfl_xor_sync_393 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 1); - float _add_392; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_393)); - p_1_1[0] = _add_392; - float _shfl_xor_sync_395 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 1); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_395)); - p_1_1[1] = _add_394; - float _shfl_xor_sync_397 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 1); - float _add_396; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_396) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_397)); - p_1_1[2] = _add_396; - float _shfl_xor_sync_399 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 1); - float _add_398; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_399)); - p_1_1[3] = _add_398; - float _add_400; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_400) : "f"(p_1_1[0]), "f"(p_1_1[2])); - float _add_401; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_401) : "f"(p_1_1[1]), "f"(p_1_1[3])); + float _fma_407; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_407) : "f"(gk_1[3]), "f"(y_1_1[3]), "f"(nz_0)); + p_1_1[3] = _fma_407; + float _shfl_xor_sync_409 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 4); + float _add_408; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_408) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_409)); + p_1_1[0] = _add_408; + float _shfl_xor_sync_411 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 4); + float _add_410; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_410) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_411)); + p_1_1[1] = _add_410; + float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 4); + float _add_412; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_412) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_413)); + p_1_1[2] = _add_412; + float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 4); + float _add_414; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_414) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_415)); + p_1_1[3] = _add_414; + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 2); + float _add_416; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_416) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_417)); + p_1_1[0] = _add_416; + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 2); + float _add_418; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_418) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_419)); + p_1_1[1] = _add_418; + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 2); + float _add_420; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_420) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_421)); + p_1_1[2] = _add_420; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 2); + float _add_422; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_422) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_423)); + p_1_1[3] = _add_422; + float _shfl_xor_sync_425 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 1); + float _add_424; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_424) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_425)); + p_1_1[0] = _add_424; + float _shfl_xor_sync_427 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 1); + float _add_426; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_426) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_427)); + p_1_1[1] = _add_426; + float _shfl_xor_sync_429 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 1); + float _add_428; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_428) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_429)); + p_1_1[2] = _add_428; + float _shfl_xor_sync_431 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 1); + float _add_430; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_430) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_431)); + p_1_1[3] = _add_430; + float _add_432; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_432) : "f"(p_1_1[0]), "f"(p_1_1[2])); + float _add_433; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_433) : "f"(p_1_1[1]), "f"(p_1_1[3])); float sq_1_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1_1) : "f"(_add_400), "f"(_add_401)); - float _shfl_sync_404 = __shfl_sync(0xFFFFFFFF, sq_1_1, 0); - float _shfl_sync_405 = __shfl_sync(0xFFFFFFFF, sq_1_1, 8); - float _add_403; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_403) : "f"(_shfl_sync_404), "f"(_shfl_sync_405)); - float _shfl_sync_406 = __shfl_sync(0xFFFFFFFF, sq_1_1, 16); - float _add_402; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_402) : "f"(_add_403), "f"(_shfl_sync_406)); - float _shfl_sync_407 = __shfl_sync(0xFFFFFFFF, sq_1_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1_1) : "f"(_add_432), "f"(_add_433)); + float _shfl_sync_436 = __shfl_sync(0xFFFFFFFF, sq_1_1, 0); + float _shfl_sync_437 = __shfl_sync(0xFFFFFFFF, sq_1_1, 8); + float _add_435; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_435) : "f"(_shfl_sync_436), "f"(_shfl_sync_437)); + float _shfl_sync_438 = __shfl_sync(0xFFFFFFFF, sq_1_1, 16); + float _add_434; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_434) : "f"(_add_435), "f"(_shfl_sync_438)); + float _shfl_sync_439 = __shfl_sync(0xFFFFFFFF, sq_1_1, 24); float dot_1_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1_1) : "f"(_add_402), "f"(_shfl_sync_407)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1_1) : "f"(_add_434), "f"(_shfl_sync_439)); float o_1_1[4]; - float _fma_409; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_409) : "f"(y_1_1[0]), "f"(dot_1_1), "f"(nz_0)); - float _sub_408; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_408) : "f"(gk_1[0]), "f"(_fma_409)); - o_1_1[0] = mul_f32_rn_noftz(rk_1[0], _sub_408); - float _fma_411; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_411) : "f"(y_1_1[1]), "f"(dot_1_1), "f"(nz_0)); - float _sub_410; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_410) : "f"(gk_1[1]), "f"(_fma_411)); - o_1_1[1] = mul_f32_rn_noftz(rk_1[0], _sub_410); - float _fma_413; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_413) : "f"(y_1_1[2]), "f"(dot_1_1), "f"(nz_0)); - float _sub_412; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_412) : "f"(gk_1[2]), "f"(_fma_413)); - o_1_1[2] = mul_f32_rn_noftz(rk_1[0], _sub_412); - float _fma_415; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_415) : "f"(y_1_1[3]), "f"(dot_1_1), "f"(nz_0)); - float _sub_414; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_414) : "f"(gk_1[3]), "f"(_fma_415)); - o_1_1[3] = mul_f32_rn_noftz(rk_1[0], _sub_414); + float _fma_441; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_441) : "f"(y_1_1[0]), "f"(dot_1_1), "f"(nz_0)); + float _sub_440; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_440) : "f"(gk_1[0]), "f"(_fma_441)); + o_1_1[0] = mul_f32_rn_noftz(rk_1[0], _sub_440); + float _fma_443; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_443) : "f"(y_1_1[1]), "f"(dot_1_1), "f"(nz_0)); + float _sub_442; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_442) : "f"(gk_1[1]), "f"(_fma_443)); + o_1_1[1] = mul_f32_rn_noftz(rk_1[0], _sub_442); + float _fma_445; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_445) : "f"(y_1_1[2]), "f"(dot_1_1), "f"(nz_0)); + float _sub_444; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_444) : "f"(gk_1[2]), "f"(_fma_445)); + o_1_1[2] = mul_f32_rn_noftz(rk_1[0], _sub_444); + float _fma_447; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_447) : "f"(y_1_1[3]), "f"(dot_1_1), "f"(nz_0)); + float _sub_446; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_446) : "f"(gk_1[3]), "f"(_fma_447)); + o_1_1[3] = mul_f32_rn_noftz(rk_1[0], _sub_446); unsigned int ob_1_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -4183,8 +4148,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; { - uint32_t _addr_416 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_416), "f"(tot_1228) : "memory"); + uint32_t _addr_448 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_448), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4202,8 +4167,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi = ((th == 1) ? tot_1228 : t_other[0]); - float t_lo = ((th == 1) ? t_other[0] : tot_1228); + float t_hi = ((th == 1) ? tot_0 : t_other[0]); + float t_lo = ((th == 1) ? t_other[0] : tot_0); float offset = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi); float sfx[8]; tmem_ld_x8(&sfx[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); @@ -4216,132 +4181,132 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_417 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_449 = make_float2(gacc[4], gacc[4]); float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_417), "l"(*(const unsigned long long*)&rp_u)); - float _abs_420; - asm("abs.f32 %0, %1;" : "=f"(_abs_420) : "f"(rp_z.x)); - float _neg_419; - asm("neg.f32 %0, %1;" : "=f"(_neg_419) : "f"(_abs_420)); - float _exp_418 = __nv_expf(_neg_419); - float _abs_423; - asm("abs.f32 %0, %1;" : "=f"(_abs_423) : "f"(rp_z.y)); - float _neg_422; - asm("neg.f32 %0, %1;" : "=f"(_neg_422) : "f"(_abs_423)); - float _exp_421 = __nv_expf(_neg_422); - float2 rp_e = make_float2(_exp_418, _exp_421); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_449), "l"(*(const unsigned long long*)&rp_u)); + float _abs_452; + asm("abs.f32 %0, %1;" : "=f"(_abs_452) : "f"(rp_z.x)); + float _neg_451; + asm("neg.f32 %0, %1;" : "=f"(_neg_451) : "f"(_abs_452)); + float _exp_450 = __nv_expf(_neg_451); + float _abs_455; + asm("abs.f32 %0, %1;" : "=f"(_abs_455) : "f"(rp_z.y)); + float _neg_454; + asm("neg.f32 %0, %1;" : "=f"(_neg_454) : "f"(_abs_455)); + float _exp_453 = __nv_expf(_neg_454); + float2 rp_e = make_float2(_exp_450, _exp_453); float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_424 = approx_rcp(rp_den.x); - float _rcp_425 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_426 = make_float2(_rcp_424, _rcp_425); + float _rcp_456 = approx_rcp(rp_den.x); + float _rcp_457 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_458 = make_float2(_rcp_456, _rcp_457); float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_426)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_427 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_459 = make_float2(gacc[5], gacc[5]); float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_427)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_459)); u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_428 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_460 = make_float2(gacc[4], gacc[4]); float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_428), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_431; - asm("abs.f32 %0, %1;" : "=f"(_abs_431) : "f"(rp_z_0.x)); - float _neg_430; - asm("neg.f32 %0, %1;" : "=f"(_neg_430) : "f"(_abs_431)); - float _exp_429 = __nv_expf(_neg_430); - float _abs_434; - asm("abs.f32 %0, %1;" : "=f"(_abs_434) : "f"(rp_z_0.y)); - float _neg_433; - asm("neg.f32 %0, %1;" : "=f"(_neg_433) : "f"(_abs_434)); - float _exp_432 = __nv_expf(_neg_433); - float2 rp_e_0 = make_float2(_exp_429, _exp_432); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_460), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_463; + asm("abs.f32 %0, %1;" : "=f"(_abs_463) : "f"(rp_z_0.x)); + float _neg_462; + asm("neg.f32 %0, %1;" : "=f"(_neg_462) : "f"(_abs_463)); + float _exp_461 = __nv_expf(_neg_462); + float _abs_466; + asm("abs.f32 %0, %1;" : "=f"(_abs_466) : "f"(rp_z_0.y)); + float _neg_465; + asm("neg.f32 %0, %1;" : "=f"(_neg_465) : "f"(_abs_466)); + float _exp_464 = __nv_expf(_neg_465); + float2 rp_e_0 = make_float2(_exp_461, _exp_464); float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_435 = approx_rcp(rp_den_0.x); - float _rcp_436 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_437 = make_float2(_rcp_435, _rcp_436); + float _rcp_467 = approx_rcp(rp_den_0.x); + float _rcp_468 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_469 = make_float2(_rcp_467, _rcp_468); float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_437)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_438 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_470 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_438)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_470)); u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_439 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_471 = make_float2(gacc[4], gacc[4]); float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_439), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_442; - asm("abs.f32 %0, %1;" : "=f"(_abs_442) : "f"(rp_z_1.x)); - float _neg_441; - asm("neg.f32 %0, %1;" : "=f"(_neg_441) : "f"(_abs_442)); - float _exp_440 = __nv_expf(_neg_441); - float _abs_445; - asm("abs.f32 %0, %1;" : "=f"(_abs_445) : "f"(rp_z_1.y)); - float _neg_444; - asm("neg.f32 %0, %1;" : "=f"(_neg_444) : "f"(_abs_445)); - float _exp_443 = __nv_expf(_neg_444); - float2 rp_e_1 = make_float2(_exp_440, _exp_443); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_471), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_474; + asm("abs.f32 %0, %1;" : "=f"(_abs_474) : "f"(rp_z_1.x)); + float _neg_473; + asm("neg.f32 %0, %1;" : "=f"(_neg_473) : "f"(_abs_474)); + float _exp_472 = __nv_expf(_neg_473); + float _abs_477; + asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(rp_z_1.y)); + float _neg_476; + asm("neg.f32 %0, %1;" : "=f"(_neg_476) : "f"(_abs_477)); + float _exp_475 = __nv_expf(_neg_476); + float2 rp_e_1 = make_float2(_exp_472, _exp_475); float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_446 = approx_rcp(rp_den_1.x); - float _rcp_447 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_448 = make_float2(_rcp_446, _rcp_447); + float _rcp_478 = approx_rcp(rp_den_1.x); + float _rcp_479 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_480 = make_float2(_rcp_478, _rcp_479); float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_448)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_480)); float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_449 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_481 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_449)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_481)); u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_450 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_482 = make_float2(gacc[4], gacc[4]); float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_450), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_453; - asm("abs.f32 %0, %1;" : "=f"(_abs_453) : "f"(rp_z_2.x)); - float _neg_452; - asm("neg.f32 %0, %1;" : "=f"(_neg_452) : "f"(_abs_453)); - float _exp_451 = __nv_expf(_neg_452); - float _abs_456; - asm("abs.f32 %0, %1;" : "=f"(_abs_456) : "f"(rp_z_2.y)); - float _neg_455; - asm("neg.f32 %0, %1;" : "=f"(_neg_455) : "f"(_abs_456)); - float _exp_454 = __nv_expf(_neg_455); - float2 rp_e_2 = make_float2(_exp_451, _exp_454); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_482), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_485; + asm("abs.f32 %0, %1;" : "=f"(_abs_485) : "f"(rp_z_2.x)); + float _neg_484; + asm("neg.f32 %0, %1;" : "=f"(_neg_484) : "f"(_abs_485)); + float _exp_483 = __nv_expf(_neg_484); + float _abs_488; + asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(rp_z_2.y)); + float _neg_487; + asm("neg.f32 %0, %1;" : "=f"(_neg_487) : "f"(_abs_488)); + float _exp_486 = __nv_expf(_neg_487); + float2 rp_e_2 = make_float2(_exp_483, _exp_486); float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_457 = approx_rcp(rp_den_2.x); - float _rcp_458 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_459 = make_float2(_rcp_457, _rcp_458); + float _rcp_489 = approx_rcp(rp_den_2.x); + float _rcp_490 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_491 = make_float2(_rcp_489, _rcp_490); float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_459)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_491)); float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_460 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_492 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_460)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_492)); u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -4356,54 +4321,54 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_461; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_461) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_461; + uint32_t _prmt_493; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_493) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_493; int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_462 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_462)); - *(reinterpret_cast(_gst_462) + (0)) = pq_0[0]; + auto _gst_494 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_494)); + *(reinterpret_cast(_gst_494) + (0)) = pq_0[0]; } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_463; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_463) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_463; + uint32_t _prmt_495; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_495) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_495; int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_464 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_464)); - *(reinterpret_cast(_gst_464) + (0)) = pq_1[0]; + auto _gst_496 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_496)); + *(reinterpret_cast(_gst_496) + (0)) = pq_1[0]; } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_465; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_465) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_465; + uint32_t _prmt_497; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_497) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_497; int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_466 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_466)); - *(reinterpret_cast(_gst_466) + (0)) = pq_2[0]; + auto _gst_498 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_498)); + *(reinterpret_cast(_gst_498) + (0)) = pq_2[0]; } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_467; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_467) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_467; + uint32_t _prmt_499; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_499) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_499; int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_468 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_468)); - *(reinterpret_cast(_gst_468) + (0)) = pq_3[0]; + auto _gst_500 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_500)); + *(reinterpret_cast(_gst_500) + (0)) = pq_3[0]; } } int t_a_0 = 48 + 8 * th; @@ -4468,47 +4433,47 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_469; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_469) : "f"(s_a), "f"(nm_t)); + float _sub_501; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_501) : "f"(s_a), "f"(nm_t)); float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_469), "f"(pa)); - float _sub_470; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_470) : "f"(pa), "f"(nm_t)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_501), "f"(pa)); + float _sub_502; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_502) : "f"(pa), "f"(nm_t)); float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_470), "f"(s_a)); - float _abs_472; - asm("abs.f32 %0, %1;" : "=f"(_abs_472) : "f"(s_a)); - float _abs_473; - asm("abs.f32 %0, %1;" : "=f"(_abs_473) : "f"(pa)); - uint32_t _setp_471_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_471_bits) : "f"(_abs_472), "f"(_abs_473)); - bool _setp_471 = _setp_471_bits != 0; - float nm_err = ((_setp_471) ? nm_es : nm_ex); - float _add_474; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_474) : "f"(c_a), "f"(nm_err)); - c_a = _add_474; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_502), "f"(s_a)); + float _abs_504; + asm("abs.f32 %0, %1;" : "=f"(_abs_504) : "f"(s_a)); + float _abs_505; + asm("abs.f32 %0, %1;" : "=f"(_abs_505) : "f"(pa)); + uint32_t _setp_503_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_503_bits) : "f"(_abs_504), "f"(_abs_505)); + bool _setp_503 = _setp_503_bits != 0; + float nm_err = ((_setp_503) ? nm_es : nm_ex); + float _add_506; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_506) : "f"(c_a), "f"(nm_err)); + c_a = _add_506; s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_475; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_475) : "f"(s_b), "f"(nm_t_0)); + float _sub_507; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_507) : "f"(s_b), "f"(nm_t_0)); float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_475), "f"(pb)); - float _sub_476; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_476) : "f"(pb), "f"(nm_t_0)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_507), "f"(pb)); + float _sub_508; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_508) : "f"(pb), "f"(nm_t_0)); float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_476), "f"(s_b)); - float _abs_478; - asm("abs.f32 %0, %1;" : "=f"(_abs_478) : "f"(s_b)); - float _abs_479; - asm("abs.f32 %0, %1;" : "=f"(_abs_479) : "f"(pb)); - uint32_t _setp_477_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_477_bits) : "f"(_abs_478), "f"(_abs_479)); - bool _setp_477 = _setp_477_bits != 0; - float nm_err_0 = ((_setp_477) ? nm_es_0 : nm_ex_0); - float _add_480; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_480) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_480; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_508), "f"(s_b)); + float _abs_510; + asm("abs.f32 %0, %1;" : "=f"(_abs_510) : "f"(s_b)); + float _abs_511; + asm("abs.f32 %0, %1;" : "=f"(_abs_511) : "f"(pb)); + uint32_t _setp_509_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_509_bits) : "f"(_abs_510), "f"(_abs_511)); + bool _setp_509 = _setp_509_bits != 0; + float nm_err_0 = ((_setp_509) ? nm_es_0 : nm_ex_0); + float _add_512; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_512) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_512; s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4532,132 +4497,132 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_481 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_513 = make_float2(gacc[4], gacc[4]); float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_481), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_484; - asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(rp_z_3.x)); - float _neg_483; - asm("neg.f32 %0, %1;" : "=f"(_neg_483) : "f"(_abs_484)); - float _exp_482 = __nv_expf(_neg_483); - float _abs_487; - asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(rp_z_3.y)); - float _neg_486; - asm("neg.f32 %0, %1;" : "=f"(_neg_486) : "f"(_abs_487)); - float _exp_485 = __nv_expf(_neg_486); - float2 rp_e_3 = make_float2(_exp_482, _exp_485); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_513), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_516; + asm("abs.f32 %0, %1;" : "=f"(_abs_516) : "f"(rp_z_3.x)); + float _neg_515; + asm("neg.f32 %0, %1;" : "=f"(_neg_515) : "f"(_abs_516)); + float _exp_514 = __nv_expf(_neg_515); + float _abs_519; + asm("abs.f32 %0, %1;" : "=f"(_abs_519) : "f"(rp_z_3.y)); + float _neg_518; + asm("neg.f32 %0, %1;" : "=f"(_neg_518) : "f"(_abs_519)); + float _exp_517 = __nv_expf(_neg_518); + float2 rp_e_3 = make_float2(_exp_514, _exp_517); float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_488 = approx_rcp(rp_den_3.x); - float _rcp_489 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_490 = make_float2(_rcp_488, _rcp_489); + float _rcp_520 = approx_rcp(rp_den_3.x); + float _rcp_521 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_522 = make_float2(_rcp_520, _rcp_521); float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_490)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_522)); float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_491 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_523 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_491)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_492 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_524 = make_float2(gacc[4], gacc[4]); float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_492), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_495; - asm("abs.f32 %0, %1;" : "=f"(_abs_495) : "f"(rp_z_4.x)); - float _neg_494; - asm("neg.f32 %0, %1;" : "=f"(_neg_494) : "f"(_abs_495)); - float _exp_493 = __nv_expf(_neg_494); - float _abs_498; - asm("abs.f32 %0, %1;" : "=f"(_abs_498) : "f"(rp_z_4.y)); - float _neg_497; - asm("neg.f32 %0, %1;" : "=f"(_neg_497) : "f"(_abs_498)); - float _exp_496 = __nv_expf(_neg_497); - float2 rp_e_4 = make_float2(_exp_493, _exp_496); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_524), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_527; + asm("abs.f32 %0, %1;" : "=f"(_abs_527) : "f"(rp_z_4.x)); + float _neg_526; + asm("neg.f32 %0, %1;" : "=f"(_neg_526) : "f"(_abs_527)); + float _exp_525 = __nv_expf(_neg_526); + float _abs_530; + asm("abs.f32 %0, %1;" : "=f"(_abs_530) : "f"(rp_z_4.y)); + float _neg_529; + asm("neg.f32 %0, %1;" : "=f"(_neg_529) : "f"(_abs_530)); + float _exp_528 = __nv_expf(_neg_529); + float2 rp_e_4 = make_float2(_exp_525, _exp_528); float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_499 = approx_rcp(rp_den_4.x); - float _rcp_500 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_501 = make_float2(_rcp_499, _rcp_500); + float _rcp_531 = approx_rcp(rp_den_4.x); + float _rcp_532 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_533 = make_float2(_rcp_531, _rcp_532); float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_501)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_533)); float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_502 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_534 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_502)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_534)); u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_503 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_535 = make_float2(gacc[4], gacc[4]); float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_503), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_506; - asm("abs.f32 %0, %1;" : "=f"(_abs_506) : "f"(rp_z_5.x)); - float _neg_505; - asm("neg.f32 %0, %1;" : "=f"(_neg_505) : "f"(_abs_506)); - float _exp_504 = __nv_expf(_neg_505); - float _abs_509; - asm("abs.f32 %0, %1;" : "=f"(_abs_509) : "f"(rp_z_5.y)); - float _neg_508; - asm("neg.f32 %0, %1;" : "=f"(_neg_508) : "f"(_abs_509)); - float _exp_507 = __nv_expf(_neg_508); - float2 rp_e_5 = make_float2(_exp_504, _exp_507); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_535), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_538; + asm("abs.f32 %0, %1;" : "=f"(_abs_538) : "f"(rp_z_5.x)); + float _neg_537; + asm("neg.f32 %0, %1;" : "=f"(_neg_537) : "f"(_abs_538)); + float _exp_536 = __nv_expf(_neg_537); + float _abs_541; + asm("abs.f32 %0, %1;" : "=f"(_abs_541) : "f"(rp_z_5.y)); + float _neg_540; + asm("neg.f32 %0, %1;" : "=f"(_neg_540) : "f"(_abs_541)); + float _exp_539 = __nv_expf(_neg_540); + float2 rp_e_5 = make_float2(_exp_536, _exp_539); float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_510 = approx_rcp(rp_den_5.x); - float _rcp_511 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_512 = make_float2(_rcp_510, _rcp_511); + float _rcp_542 = approx_rcp(rp_den_5.x); + float _rcp_543 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_544 = make_float2(_rcp_542, _rcp_543); float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_512)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_544)); float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_513 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_545 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_513)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_545)); u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_514 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_546 = make_float2(gacc[4], gacc[4]); float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_514), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_517; - asm("abs.f32 %0, %1;" : "=f"(_abs_517) : "f"(rp_z_6.x)); - float _neg_516; - asm("neg.f32 %0, %1;" : "=f"(_neg_516) : "f"(_abs_517)); - float _exp_515 = __nv_expf(_neg_516); - float _abs_520; - asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_6.y)); - float _neg_519; - asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); - float _exp_518 = __nv_expf(_neg_519); - float2 rp_e_6 = make_float2(_exp_515, _exp_518); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_546), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_549; + asm("abs.f32 %0, %1;" : "=f"(_abs_549) : "f"(rp_z_6.x)); + float _neg_548; + asm("neg.f32 %0, %1;" : "=f"(_neg_548) : "f"(_abs_549)); + float _exp_547 = __nv_expf(_neg_548); + float _abs_552; + asm("abs.f32 %0, %1;" : "=f"(_abs_552) : "f"(rp_z_6.y)); + float _neg_551; + asm("neg.f32 %0, %1;" : "=f"(_neg_551) : "f"(_abs_552)); + float _exp_550 = __nv_expf(_neg_551); + float2 rp_e_6 = make_float2(_exp_547, _exp_550); float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_521 = approx_rcp(rp_den_6.x); - float _rcp_522 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_523 = make_float2(_rcp_521, _rcp_522); + float _rcp_553 = approx_rcp(rp_den_6.x); + float _rcp_554 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_555 = make_float2(_rcp_553, _rcp_554); float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_555)); float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_524 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_556 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_524)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_556)); u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4672,54 +4637,54 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_525; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_525) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_525; + uint32_t _prmt_557; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_557) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_557; int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_526 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_526)); - *(reinterpret_cast(_gst_526) + (0)) = pq_4[0]; + auto _gst_558 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_558)); + *(reinterpret_cast(_gst_558) + (0)) = pq_4[0]; } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_527; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_527) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_527; + uint32_t _prmt_559; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_559) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_559; int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_528 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_528)); - *(reinterpret_cast(_gst_528) + (0)) = pq_5[0]; + auto _gst_560 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_560)); + *(reinterpret_cast(_gst_560) + (0)) = pq_5[0]; } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_529; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_529) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_529; + uint32_t _prmt_561; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_561; int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_530 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_530)); - *(reinterpret_cast(_gst_530) + (0)) = pq_6[0]; + auto _gst_562 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_562)); + *(reinterpret_cast(_gst_562) + (0)) = pq_6[0]; } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_531; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_531) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_531; + uint32_t _prmt_563; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_563; int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_532 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_532)); - *(reinterpret_cast(_gst_532) + (0)) = pq_7[0]; + auto _gst_564 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_564)); + *(reinterpret_cast(_gst_564) + (0)) = pq_7[0]; } } int t_a_8 = 32 + 8 * th; @@ -4784,47 +4749,47 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_533; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_533) : "f"(s_a_0), "f"(nm_t_1)); + float _sub_565; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_565) : "f"(s_a_0), "f"(nm_t_1)); float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_533), "f"(pa_0)); - float _sub_534; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_534) : "f"(pa_0), "f"(nm_t_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_565), "f"(pa_0)); + float _sub_566; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_566) : "f"(pa_0), "f"(nm_t_1)); float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_534), "f"(s_a_0)); - float _abs_536; - asm("abs.f32 %0, %1;" : "=f"(_abs_536) : "f"(s_a_0)); - float _abs_537; - asm("abs.f32 %0, %1;" : "=f"(_abs_537) : "f"(pa_0)); - uint32_t _setp_535_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_535_bits) : "f"(_abs_536), "f"(_abs_537)); - bool _setp_535 = _setp_535_bits != 0; - float nm_err_1 = ((_setp_535) ? nm_es_1 : nm_ex_1); - float _add_538; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_538) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_538; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_566), "f"(s_a_0)); + float _abs_568; + asm("abs.f32 %0, %1;" : "=f"(_abs_568) : "f"(s_a_0)); + float _abs_569; + asm("abs.f32 %0, %1;" : "=f"(_abs_569) : "f"(pa_0)); + uint32_t _setp_567_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_567_bits) : "f"(_abs_568), "f"(_abs_569)); + bool _setp_567 = _setp_567_bits != 0; + float nm_err_1 = ((_setp_567) ? nm_es_1 : nm_ex_1); + float _add_570; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_570; s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_539; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_539) : "f"(s_b_0), "f"(nm_t_2)); + float _sub_571; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(s_b_0), "f"(nm_t_2)); float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_539), "f"(pb_0)); - float _sub_540; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_540) : "f"(pb_0), "f"(nm_t_2)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_571), "f"(pb_0)); + float _sub_572; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_572) : "f"(pb_0), "f"(nm_t_2)); float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_540), "f"(s_b_0)); - float _abs_542; - asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(s_b_0)); - float _abs_543; - asm("abs.f32 %0, %1;" : "=f"(_abs_543) : "f"(pb_0)); - uint32_t _setp_541_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_541_bits) : "f"(_abs_542), "f"(_abs_543)); - bool _setp_541 = _setp_541_bits != 0; - float nm_err_2 = ((_setp_541) ? nm_es_2 : nm_ex_2); - float _add_544; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_544) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_544; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_572), "f"(s_b_0)); + float _abs_574; + asm("abs.f32 %0, %1;" : "=f"(_abs_574) : "f"(s_b_0)); + float _abs_575; + asm("abs.f32 %0, %1;" : "=f"(_abs_575) : "f"(pb_0)); + uint32_t _setp_573_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_573_bits) : "f"(_abs_574), "f"(_abs_575)); + bool _setp_573 = _setp_573_bits != 0; + float nm_err_2 = ((_setp_573) ? nm_es_2 : nm_ex_2); + float _add_576; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_576; s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4832,8 +4797,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_545 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_545), "f"(off_acc[0]) : "memory"); + uint32_t _addr_577 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_577), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4859,342 +4824,155 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1292 = r_297[0]; - float gc_1293 = gn_1292 - 21.640425f; - int eb_1294 = 81920 + th * -32768; - int ew_1295 = eb_1294 >> 2; - int qbase_1296 = pool_f32_addr + (unsigned int)(eb_1294 + 12288 + dh_lane * 1024); - int sb8_1297 = 16 + 8 * th; - int gb8_1298 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1297 * 128); - int kb8_1299 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1297 * 128); - float dbp_1300[8]; - float _tmem_load_20[8]; - tmem_ld_x8(&_tmem_load_20[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); - float _tmem_load_21[8]; - tmem_ld_x8(&_tmem_load_21[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); - float _tmem_load_22[8]; - tmem_ld_x8(&_tmem_load_22[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); - float _tmem_load_23[8]; - tmem_ld_x8(&_tmem_load_23[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); - float _tmem_load_24[8]; - tmem_ld_x8(&_tmem_load_24[0], taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th)); - float btv_1301[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1301[0])), "=r"(*reinterpret_cast(&btv_1301[(0) + 1])), "=r"(*reinterpret_cast(&btv_1301[(0) + 2])), "=r"(*reinterpret_cast(&btv_1301[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1301[4])), "=r"(*reinterpret_cast(&btv_1301[(4) + 1])), "=r"(*reinterpret_cast(&btv_1301[(4) + 2])), "=r"(*reinterpret_cast(&btv_1301[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th) + 16))); + float r_276[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1 = r_276[0]; + float gc_1 = gn_1 - 21.640425f; + int eb_1 = 81920 + th * -32768; + int ew_3 = eb_1 >> 2; + int qbase_1 = pool_f32_addr + (unsigned int)(eb_1 + 12288 + dh_lane * 1024); + int sb8_1 = 16 + 8 * th; + int gb8_1 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1 * 128); + int kb8_1 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1 * 128); + int c_qa_0 = 64 + 8 * th; + int c_ka_0 = 80 + 8 * th; + int c_qd_0 = 96 + 8 * th; + int c_kd_0 = 112 + 8 * th; + int c_dk_0 = 16 + 8 * th; + int c_bp_0 = c_ka_0; + float s_run_1[1]; + s_run_1[0] = 0.0f; + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_3 = 7 - k_1; + float a_qd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_3))); + float a_kd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_0 + j_3))); + float a_dk_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_0 + j_3))); + float a_qa_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa_0 + j_3))); + float a_ka_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka_0 + j_3))); + float btr_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_1 + j_3)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_1 + j_3; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 * j_3 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_3 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 * j_3 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_3 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_3 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_3 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1 = __uint_as_float(r_279[0] << 16); + float q_m_1 = ((t_1 >= clen) ? 0.0f : q_t_1); + float dqf_1 = pool_f32[ew_3 + 128 * j_3 + d_lane]; + float dkf_1 = pool_f32[ew_3 + (1024 + 128 * j_3) + d_lane]; + float dgf_1 = pool_f32[ew_3 + (2048 + 128 * j_3) + d_lane]; + float bt_t_1 = btr_1[0]; + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float e_off_1 = approx_exp2(g_t_1 - gn_1); + float dq2_1 = __fmaf_rn(a_qa_1[0], e_off_1, a_qd_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(a_ka_1[0], e_off_1, a_kd_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + a_dk_1[0]; + float qe_8 = q_m_1 * e_dg_1; + float kp_20 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_1; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, a_qd_1[0], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_1) * a_kd_1[0]); + float dg_o_1 = __fmaf_rn(q_m_1, dq2_1, __fmaf_rn(dk2b_1 - a_dk_1[0], k_t_1, cr_8)) + dgf_1; + s_run_1[0] = s_run_1[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_3)), "f"(s_run_1[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_3)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(81920 - 32768 * th >> 2) + (128 * j_3 + d_lane)] = dq_o_1; + pool_f32[(81920 - 32768 * th >> 2) + (1024 + 128 * j_3 + d_lane)] = dk_o_1; + } + float tot_1 = s_run_1[0]; + float dbp_1[8]; + tmem_ld_x8(&dbp_1[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1302[1]; - s_run_1302[0] = 0.0f; - int t_8 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_1295 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_1295 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_1295 + 2944 + d_lane]; - float bt_t_8 = btv_1301[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_1292); - float e_dg_8 = approx_exp2(g_t_8 - gc_1293); - float dq2_8 = __fmaf_rn(_tmem_load_20[7], e_off_8, _tmem_load_22[7] * e_dg_8); - float dk2_8 = __fmaf_rn(_tmem_load_21[7], e_off_8, _tmem_load_23[7] * e_dg_8); - dbp_1300[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_24[7]; - float dg_o_8 = __fmaf_rn(q_t_1_16, dq2_8, (dk2b_8 - _tmem_load_24[7]) * k_t_8) + dgf_8; - s_run_1302[0] = s_run_1302[0] + dg_o_8; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_8; - pool_f32[(81920 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_8; - int t_9 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_1295 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_1295 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_1295 + 2816 + d_lane]; - float bt_t_9 = btv_1301[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_1292); - float e_dg_9 = approx_exp2(g_t_9 - gc_1293); - float dq2_9 = __fmaf_rn(_tmem_load_20[6], e_off_9, _tmem_load_22[6] * e_dg_9); - float dk2_9 = __fmaf_rn(_tmem_load_21[6], e_off_9, _tmem_load_23[6] * e_dg_9); - dbp_1300[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[6]; - float dg_o_9 = __fmaf_rn(q_t_1_17, dq2_9, (dk2b_9 - _tmem_load_24[6]) * k_t_9) + dgf_9; - s_run_1302[0] = s_run_1302[0] + dg_o_9; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_9; - pool_f32[(81920 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_9; - int t_10 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_1295 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_1295 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_1295 + 2688 + d_lane]; - float bt_t_10 = btv_1301[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_1292); - float e_dg_10 = approx_exp2(g_t_10 - gc_1293); - float dq2_10 = __fmaf_rn(_tmem_load_20[5], e_off_10, _tmem_load_22[5] * e_dg_10); - float dk2_10 = __fmaf_rn(_tmem_load_21[5], e_off_10, _tmem_load_23[5] * e_dg_10); - dbp_1300[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[5]; - float dg_o_10 = __fmaf_rn(q_t_1_18, dq2_10, (dk2b_10 - _tmem_load_24[5]) * k_t_10) + dgf_10; - s_run_1302[0] = s_run_1302[0] + dg_o_10; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_10; - pool_f32[(81920 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_10; - int t_11 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_1295 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_1295 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_1295 + 2560 + d_lane]; - float bt_t_11 = btv_1301[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_1292); - float e_dg_11 = approx_exp2(g_t_11 - gc_1293); - float dq2_11 = __fmaf_rn(_tmem_load_20[4], e_off_11, _tmem_load_22[4] * e_dg_11); - float dk2_11 = __fmaf_rn(_tmem_load_21[4], e_off_11, _tmem_load_23[4] * e_dg_11); - dbp_1300[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[4]; - float dg_o_11 = __fmaf_rn(q_t_1_19, dq2_11, (dk2b_11 - _tmem_load_24[4]) * k_t_11) + dgf_11; - s_run_1302[0] = s_run_1302[0] + dg_o_11; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_11; - pool_f32[(81920 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_11; - int t_12 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_1295 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_1295 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_1295 + 2432 + d_lane]; - float bt_t_12 = btv_1301[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_1292); - float e_dg_12 = approx_exp2(g_t_12 - gc_1293); - float dq2_12 = __fmaf_rn(_tmem_load_20[3], e_off_12, _tmem_load_22[3] * e_dg_12); - float dk2_12 = __fmaf_rn(_tmem_load_21[3], e_off_12, _tmem_load_23[3] * e_dg_12); - dbp_1300[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[3]; - float dg_o_12 = __fmaf_rn(q_t_1_20, dq2_12, (dk2b_12 - _tmem_load_24[3]) * k_t_12) + dgf_12; - s_run_1302[0] = s_run_1302[0] + dg_o_12; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_12; - pool_f32[(81920 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_12; - int t_13 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_1295 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_1295 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_1295 + 2304 + d_lane]; - float bt_t_13 = btv_1301[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_1292); - float e_dg_13 = approx_exp2(g_t_13 - gc_1293); - float dq2_13 = __fmaf_rn(_tmem_load_20[2], e_off_13, _tmem_load_22[2] * e_dg_13); - float dk2_13 = __fmaf_rn(_tmem_load_21[2], e_off_13, _tmem_load_23[2] * e_dg_13); - dbp_1300[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[2]; - float dg_o_13 = __fmaf_rn(q_t_1_21, dq2_13, (dk2b_13 - _tmem_load_24[2]) * k_t_13) + dgf_13; - s_run_1302[0] = s_run_1302[0] + dg_o_13; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_13; - pool_f32[(81920 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_13; - int t_14 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_1295 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_1295 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_1295 + 2176 + d_lane]; - float bt_t_14 = btv_1301[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_1292); - float e_dg_14 = approx_exp2(g_t_14 - gc_1293); - float dq2_14 = __fmaf_rn(_tmem_load_20[1], e_off_14, _tmem_load_22[1] * e_dg_14); - float dk2_14 = __fmaf_rn(_tmem_load_21[1], e_off_14, _tmem_load_23[1] * e_dg_14); - dbp_1300[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[1]; - float dg_o_14 = __fmaf_rn(q_t_1_22, dq2_14, (dk2b_14 - _tmem_load_24[1]) * k_t_14) + dgf_14; - s_run_1302[0] = s_run_1302[0] + dg_o_14; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_14; - pool_f32[(81920 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_14; - int t_15 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_1295 + d_lane]; - float dkf_15 = pool_f32[ew_1295 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_1295 + 2048 + d_lane]; - float bt_t_15 = btv_1301[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_1292); - float e_dg_15 = approx_exp2(g_t_15 - gc_1293); - float dq2_15 = __fmaf_rn(_tmem_load_20[0], e_off_15, _tmem_load_22[0] * e_dg_15); - float dk2_15 = __fmaf_rn(_tmem_load_21[0], e_off_15, _tmem_load_23[0] * e_dg_15); - dbp_1300[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[0]; - float dg_o_15 = __fmaf_rn(q_t_1_23, dq2_15, (dk2b_15 - _tmem_load_24[0]) * k_t_15) + dgf_15; - s_run_1302[0] = s_run_1302[0] + dg_o_15; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + d_lane] = dq_o_15; - pool_f32[(81920 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_15; - float tot_1471 = s_run_1302[0]; - int b4_1472 = lane_1 >> 4 & 1; - int b3_1473 = lane_1 >> 3 & 1; - int b2_1474 = lane_1 >> 2 & 1; - float v4_1475[4]; - float send_6 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_546 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_546; - float send_7 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_547 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_547; - float send_8 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_548 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_548; - float send_9 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_549 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_549; - float v2_1480[2]; - float send_10 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_550 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_550; - float send_11 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_551 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_551; - float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_552 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_552; - float _shfl_xor_sync_553 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_553; - float _shfl_xor_sync_554 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_554; + int b4_1 = lane_1 >> 4 & 1; + int b3_1 = lane_1 >> 3 & 1; + int b2_1 = lane_1 >> 2 & 1; + float v4_1[4]; + float send_4 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); + float _shfl_xor_sync_578 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_578; + float send_5 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); + float _shfl_xor_sync_579 = __shfl_xor_sync(0xFFFFFFFF, send_5, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_579; + float send_6 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); + float _shfl_xor_sync_580 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_580; + float send_7 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); + float _shfl_xor_sync_581 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_581; + float v2_16[2]; + float send2_2 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); + float _shfl_xor_sync_582 = __shfl_xor_sync(0xFFFFFFFF, send2_2, 8); + v2_16[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_582; + float send2_3 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); + float _shfl_xor_sync_583 = __shfl_xor_sync(0xFFFFFFFF, send2_3, 8); + v2_16[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_583; + float send1_0 = ((b2_1 == 0) ? v2_16[1] : v2_16[0]); + float _shfl_xor_sync_584 = __shfl_xor_sync(0xFFFFFFFF, send1_0, 4); + float v1_1 = ((b2_1 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_584; + float _shfl_xor_sync_585 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_585; + float _shfl_xor_sync_586 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_586; asm volatile("tcgen05.fence::before_thread_sync;"); - int ew_2 = 81920 - 32768 * th >> 2; + int ew_4 = 81920 - 32768 * th >> 2; float nz_1 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); asm volatile("barrier.sync 1, 256;" ::: "memory"); #pragma unroll 1 for (int i_2 = 0; i_2 < 2; i_2++) { - int j_2 = qd + 4 * i_2; - bool live_2 = 16 + 8 * th + j_2 < clen; - int _min_555 = ((16 + 8 * th + j_2) < (clen - 1) ? (16 + 8 * th + j_2) : (clen - 1)); - long long row_2 = ext0 + (long long)_min_555; + int j_4 = qd + 4 * i_2; + bool live_2 = 16 + 8 * th + j_4 < clen; + int _min_587 = ((16 + 8 * th + j_4) < (clen - 1) ? (16 + 8 * th + j_4) : (clen - 1)); + long long row_2 = ext0 + (long long)_min_587; float gq_2[4]; float gk_2[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gq_2[0])), "=r"(*reinterpret_cast(&gq_2[(0) + 1])), "=r"(*reinterpret_cast(&gq_2[(0) + 2])), "=r"(*reinterpret_cast(&gq_2[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_2 + 128 * j_2 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_4 + 128 * j_4 + 4 * lane_1)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gk_2[0])), "=r"(*reinterpret_cast(&gk_2[(0) + 1])), "=r"(*reinterpret_cast(&gk_2[(0) + 2])), "=r"(*reinterpret_cast(&gk_2[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_2 + 1024 + 128 * j_2 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_4 + 1024 + 128 * j_4 + 4 * lane_1)))); if (i_2 == 1) { mbarrier_arrive(epi_empty_addr + (2 + th) * 8); } @@ -5203,26 +4981,26 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float rq_2[1]; float rk_2[1]; { - uint2 _vld_556; - _vld_556 = *reinterpret_cast(q_raw + (row_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_556 = reinterpret_cast(&_vld_556); + uint2 _vld_588; + _vld_588 = *reinterpret_cast(q_raw + (row_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_588 = reinterpret_cast(&_vld_588); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xq_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_556[_pair]) << 16); - (&xq_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_556[_pair]) & 0xffff0000u); + (&xq_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_588[_pair]) << 16); + (&xq_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_588[_pair]) & 0xffff0000u); } } { rq_2[0] = *reinterpret_cast(qk_rstd + ((0 + row_2) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_557; - _vld_557 = *reinterpret_cast(k_raw + (row_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_557 = reinterpret_cast(&_vld_557); + uint2 _vld_589; + _vld_589 = *reinterpret_cast(k_raw + (row_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_589 = reinterpret_cast(&_vld_589); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xk_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_557[_pair]) << 16); - (&xk_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_557[_pair]) & 0xffff0000u); + (&xk_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_589[_pair]) << 16); + (&xk_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_589[_pair]) & 0xffff0000u); } } { @@ -5232,106 +5010,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_2[4]; float p_2[4]; y_2[0] = mul_f32_rn_noftz(xq_2[0], rq_2[0]); - float _fma_558; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_558) : "f"(gq_2[0]), "f"(y_2[0]), "f"(nz_1)); - p_2[0] = _fma_558; + float _fma_590; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_590) : "f"(gq_2[0]), "f"(y_2[0]), "f"(nz_1)); + p_2[0] = _fma_590; y_2[1] = mul_f32_rn_noftz(xq_2[1], rq_2[0]); - float _fma_559; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_559) : "f"(gq_2[1]), "f"(y_2[1]), "f"(nz_1)); - p_2[1] = _fma_559; + float _fma_591; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_591) : "f"(gq_2[1]), "f"(y_2[1]), "f"(nz_1)); + p_2[1] = _fma_591; y_2[2] = mul_f32_rn_noftz(xq_2[2], rq_2[0]); - float _fma_560; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_560) : "f"(gq_2[2]), "f"(y_2[2]), "f"(nz_1)); - p_2[2] = _fma_560; + float _fma_592; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_592) : "f"(gq_2[2]), "f"(y_2[2]), "f"(nz_1)); + p_2[2] = _fma_592; y_2[3] = mul_f32_rn_noftz(xq_2[3], rq_2[0]); - float _fma_561; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_561) : "f"(gq_2[3]), "f"(y_2[3]), "f"(nz_1)); - p_2[3] = _fma_561; - float _shfl_xor_sync_563 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 4); - float _add_562; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_562) : "f"(p_2[0]), "f"(_shfl_xor_sync_563)); - p_2[0] = _add_562; - float _shfl_xor_sync_565 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 4); - float _add_564; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_564) : "f"(p_2[1]), "f"(_shfl_xor_sync_565)); - p_2[1] = _add_564; - float _shfl_xor_sync_567 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 4); - float _add_566; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_566) : "f"(p_2[2]), "f"(_shfl_xor_sync_567)); - p_2[2] = _add_566; - float _shfl_xor_sync_569 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 4); - float _add_568; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_568) : "f"(p_2[3]), "f"(_shfl_xor_sync_569)); - p_2[3] = _add_568; - float _shfl_xor_sync_571 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 2); - float _add_570; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(p_2[0]), "f"(_shfl_xor_sync_571)); - p_2[0] = _add_570; - float _shfl_xor_sync_573 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 2); - float _add_572; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_572) : "f"(p_2[1]), "f"(_shfl_xor_sync_573)); - p_2[1] = _add_572; - float _shfl_xor_sync_575 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 2); - float _add_574; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(p_2[2]), "f"(_shfl_xor_sync_575)); - p_2[2] = _add_574; - float _shfl_xor_sync_577 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 2); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(p_2[3]), "f"(_shfl_xor_sync_577)); - p_2[3] = _add_576; - float _shfl_xor_sync_579 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 1); - float _add_578; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_578) : "f"(p_2[0]), "f"(_shfl_xor_sync_579)); - p_2[0] = _add_578; - float _shfl_xor_sync_581 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 1); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(p_2[1]), "f"(_shfl_xor_sync_581)); - p_2[1] = _add_580; - float _shfl_xor_sync_583 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 1); - float _add_582; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_582) : "f"(p_2[2]), "f"(_shfl_xor_sync_583)); - p_2[2] = _add_582; - float _shfl_xor_sync_585 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 1); - float _add_584; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(p_2[3]), "f"(_shfl_xor_sync_585)); - p_2[3] = _add_584; - float _add_586; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_586) : "f"(p_2[0]), "f"(p_2[2])); - float _add_587; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_587) : "f"(p_2[1]), "f"(p_2[3])); + float _fma_593; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_593) : "f"(gq_2[3]), "f"(y_2[3]), "f"(nz_1)); + p_2[3] = _fma_593; + float _shfl_xor_sync_595 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 4); + float _add_594; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_594) : "f"(p_2[0]), "f"(_shfl_xor_sync_595)); + p_2[0] = _add_594; + float _shfl_xor_sync_597 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 4); + float _add_596; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_596) : "f"(p_2[1]), "f"(_shfl_xor_sync_597)); + p_2[1] = _add_596; + float _shfl_xor_sync_599 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 4); + float _add_598; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_598) : "f"(p_2[2]), "f"(_shfl_xor_sync_599)); + p_2[2] = _add_598; + float _shfl_xor_sync_601 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 4); + float _add_600; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_600) : "f"(p_2[3]), "f"(_shfl_xor_sync_601)); + p_2[3] = _add_600; + float _shfl_xor_sync_603 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 2); + float _add_602; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_602) : "f"(p_2[0]), "f"(_shfl_xor_sync_603)); + p_2[0] = _add_602; + float _shfl_xor_sync_605 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 2); + float _add_604; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_604) : "f"(p_2[1]), "f"(_shfl_xor_sync_605)); + p_2[1] = _add_604; + float _shfl_xor_sync_607 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 2); + float _add_606; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_606) : "f"(p_2[2]), "f"(_shfl_xor_sync_607)); + p_2[2] = _add_606; + float _shfl_xor_sync_609 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 2); + float _add_608; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_608) : "f"(p_2[3]), "f"(_shfl_xor_sync_609)); + p_2[3] = _add_608; + float _shfl_xor_sync_611 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 1); + float _add_610; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_610) : "f"(p_2[0]), "f"(_shfl_xor_sync_611)); + p_2[0] = _add_610; + float _shfl_xor_sync_613 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 1); + float _add_612; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_612) : "f"(p_2[1]), "f"(_shfl_xor_sync_613)); + p_2[1] = _add_612; + float _shfl_xor_sync_615 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 1); + float _add_614; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_614) : "f"(p_2[2]), "f"(_shfl_xor_sync_615)); + p_2[2] = _add_614; + float _shfl_xor_sync_617 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 1); + float _add_616; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_616) : "f"(p_2[3]), "f"(_shfl_xor_sync_617)); + p_2[3] = _add_616; + float _add_618; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_618) : "f"(p_2[0]), "f"(p_2[2])); + float _add_619; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_619) : "f"(p_2[1]), "f"(p_2[3])); float sq_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2) : "f"(_add_586), "f"(_add_587)); - float _shfl_sync_590 = __shfl_sync(0xFFFFFFFF, sq_2, 0); - float _shfl_sync_591 = __shfl_sync(0xFFFFFFFF, sq_2, 8); - float _add_589; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_589) : "f"(_shfl_sync_590), "f"(_shfl_sync_591)); - float _shfl_sync_592 = __shfl_sync(0xFFFFFFFF, sq_2, 16); - float _add_588; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_588) : "f"(_add_589), "f"(_shfl_sync_592)); - float _shfl_sync_593 = __shfl_sync(0xFFFFFFFF, sq_2, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2) : "f"(_add_618), "f"(_add_619)); + float _shfl_sync_622 = __shfl_sync(0xFFFFFFFF, sq_2, 0); + float _shfl_sync_623 = __shfl_sync(0xFFFFFFFF, sq_2, 8); + float _add_621; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_621) : "f"(_shfl_sync_622), "f"(_shfl_sync_623)); + float _shfl_sync_624 = __shfl_sync(0xFFFFFFFF, sq_2, 16); + float _add_620; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_620) : "f"(_add_621), "f"(_shfl_sync_624)); + float _shfl_sync_625 = __shfl_sync(0xFFFFFFFF, sq_2, 24); float dot_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_588), "f"(_shfl_sync_593)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_620), "f"(_shfl_sync_625)); float o_2[4]; - float _fma_595; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_595) : "f"(y_2[0]), "f"(dot_2), "f"(nz_1)); - float _sub_594; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_594) : "f"(gq_2[0]), "f"(_fma_595)); - o_2[0] = mul_f32_rn_noftz(rq_2[0], _sub_594); - float _fma_597; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_597) : "f"(y_2[1]), "f"(dot_2), "f"(nz_1)); - float _sub_596; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_596) : "f"(gq_2[1]), "f"(_fma_597)); - o_2[1] = mul_f32_rn_noftz(rq_2[0], _sub_596); - float _fma_599; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_599) : "f"(y_2[2]), "f"(dot_2), "f"(nz_1)); - float _sub_598; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_598) : "f"(gq_2[2]), "f"(_fma_599)); - o_2[2] = mul_f32_rn_noftz(rq_2[0], _sub_598); - float _fma_601; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_601) : "f"(y_2[3]), "f"(dot_2), "f"(nz_1)); - float _sub_600; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_600) : "f"(gq_2[3]), "f"(_fma_601)); - o_2[3] = mul_f32_rn_noftz(rq_2[0], _sub_600); + float _fma_627; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_627) : "f"(y_2[0]), "f"(dot_2), "f"(nz_1)); + float _sub_626; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_626) : "f"(gq_2[0]), "f"(_fma_627)); + o_2[0] = mul_f32_rn_noftz(rq_2[0], _sub_626); + float _fma_629; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_629) : "f"(y_2[1]), "f"(dot_2), "f"(nz_1)); + float _sub_628; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_628) : "f"(gq_2[1]), "f"(_fma_629)); + o_2[1] = mul_f32_rn_noftz(rq_2[0], _sub_628); + float _fma_631; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_631) : "f"(y_2[2]), "f"(dot_2), "f"(nz_1)); + float _sub_630; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_630) : "f"(gq_2[2]), "f"(_fma_631)); + o_2[2] = mul_f32_rn_noftz(rq_2[0], _sub_630); + float _fma_633; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_633) : "f"(y_2[3]), "f"(dot_2), "f"(nz_1)); + float _sub_632; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_632) : "f"(gq_2[3]), "f"(_fma_633)); + o_2[3] = mul_f32_rn_noftz(rq_2[0], _sub_632); unsigned int ob_2[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -5342,106 +5120,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_2_1[4]; float p_2_1[4]; y_2_1[0] = mul_f32_rn_noftz(xk_2[0], rk_2[0]); - float _fma_602; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_602) : "f"(gk_2[0]), "f"(y_2_1[0]), "f"(nz_1)); - p_2_1[0] = _fma_602; + float _fma_634; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_634) : "f"(gk_2[0]), "f"(y_2_1[0]), "f"(nz_1)); + p_2_1[0] = _fma_634; y_2_1[1] = mul_f32_rn_noftz(xk_2[1], rk_2[0]); - float _fma_603; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_603) : "f"(gk_2[1]), "f"(y_2_1[1]), "f"(nz_1)); - p_2_1[1] = _fma_603; + float _fma_635; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_635) : "f"(gk_2[1]), "f"(y_2_1[1]), "f"(nz_1)); + p_2_1[1] = _fma_635; y_2_1[2] = mul_f32_rn_noftz(xk_2[2], rk_2[0]); - float _fma_604; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_604) : "f"(gk_2[2]), "f"(y_2_1[2]), "f"(nz_1)); - p_2_1[2] = _fma_604; + float _fma_636; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_636) : "f"(gk_2[2]), "f"(y_2_1[2]), "f"(nz_1)); + p_2_1[2] = _fma_636; y_2_1[3] = mul_f32_rn_noftz(xk_2[3], rk_2[0]); - float _fma_605; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_605) : "f"(gk_2[3]), "f"(y_2_1[3]), "f"(nz_1)); - p_2_1[3] = _fma_605; - float _shfl_xor_sync_607 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 4); - float _add_606; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_606) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_607)); - p_2_1[0] = _add_606; - float _shfl_xor_sync_609 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 4); - float _add_608; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_608) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_609)); - p_2_1[1] = _add_608; - float _shfl_xor_sync_611 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 4); - float _add_610; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_610) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_611)); - p_2_1[2] = _add_610; - float _shfl_xor_sync_613 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 4); - float _add_612; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_612) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_613)); - p_2_1[3] = _add_612; - float _shfl_xor_sync_615 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 2); - float _add_614; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_614) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_615)); - p_2_1[0] = _add_614; - float _shfl_xor_sync_617 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 2); - float _add_616; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_616) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_617)); - p_2_1[1] = _add_616; - float _shfl_xor_sync_619 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 2); - float _add_618; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_618) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_619)); - p_2_1[2] = _add_618; - float _shfl_xor_sync_621 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 2); - float _add_620; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_620) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_621)); - p_2_1[3] = _add_620; - float _shfl_xor_sync_623 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 1); - float _add_622; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_622) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_623)); - p_2_1[0] = _add_622; - float _shfl_xor_sync_625 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 1); - float _add_624; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_624) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_625)); - p_2_1[1] = _add_624; - float _shfl_xor_sync_627 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 1); - float _add_626; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_626) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_627)); - p_2_1[2] = _add_626; - float _shfl_xor_sync_629 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 1); - float _add_628; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_628) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_629)); - p_2_1[3] = _add_628; - float _add_630; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_630) : "f"(p_2_1[0]), "f"(p_2_1[2])); - float _add_631; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_631) : "f"(p_2_1[1]), "f"(p_2_1[3])); + float _fma_637; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_637) : "f"(gk_2[3]), "f"(y_2_1[3]), "f"(nz_1)); + p_2_1[3] = _fma_637; + float _shfl_xor_sync_639 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 4); + float _add_638; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_638) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_639)); + p_2_1[0] = _add_638; + float _shfl_xor_sync_641 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 4); + float _add_640; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_640) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_641)); + p_2_1[1] = _add_640; + float _shfl_xor_sync_643 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 4); + float _add_642; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_642) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_643)); + p_2_1[2] = _add_642; + float _shfl_xor_sync_645 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 4); + float _add_644; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_644) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_645)); + p_2_1[3] = _add_644; + float _shfl_xor_sync_647 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 2); + float _add_646; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_646) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_647)); + p_2_1[0] = _add_646; + float _shfl_xor_sync_649 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 2); + float _add_648; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_648) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_649)); + p_2_1[1] = _add_648; + float _shfl_xor_sync_651 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 2); + float _add_650; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_650) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_651)); + p_2_1[2] = _add_650; + float _shfl_xor_sync_653 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 2); + float _add_652; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_652) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_653)); + p_2_1[3] = _add_652; + float _shfl_xor_sync_655 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 1); + float _add_654; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_654) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_655)); + p_2_1[0] = _add_654; + float _shfl_xor_sync_657 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 1); + float _add_656; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_656) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_657)); + p_2_1[1] = _add_656; + float _shfl_xor_sync_659 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 1); + float _add_658; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_658) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_659)); + p_2_1[2] = _add_658; + float _shfl_xor_sync_661 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 1); + float _add_660; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_660) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_661)); + p_2_1[3] = _add_660; + float _add_662; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_662) : "f"(p_2_1[0]), "f"(p_2_1[2])); + float _add_663; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_663) : "f"(p_2_1[1]), "f"(p_2_1[3])); float sq_2_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2_1) : "f"(_add_630), "f"(_add_631)); - float _shfl_sync_634 = __shfl_sync(0xFFFFFFFF, sq_2_1, 0); - float _shfl_sync_635 = __shfl_sync(0xFFFFFFFF, sq_2_1, 8); - float _add_633; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_633) : "f"(_shfl_sync_634), "f"(_shfl_sync_635)); - float _shfl_sync_636 = __shfl_sync(0xFFFFFFFF, sq_2_1, 16); - float _add_632; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_632) : "f"(_add_633), "f"(_shfl_sync_636)); - float _shfl_sync_637 = __shfl_sync(0xFFFFFFFF, sq_2_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2_1) : "f"(_add_662), "f"(_add_663)); + float _shfl_sync_666 = __shfl_sync(0xFFFFFFFF, sq_2_1, 0); + float _shfl_sync_667 = __shfl_sync(0xFFFFFFFF, sq_2_1, 8); + float _add_665; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_665) : "f"(_shfl_sync_666), "f"(_shfl_sync_667)); + float _shfl_sync_668 = __shfl_sync(0xFFFFFFFF, sq_2_1, 16); + float _add_664; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_664) : "f"(_add_665), "f"(_shfl_sync_668)); + float _shfl_sync_669 = __shfl_sync(0xFFFFFFFF, sq_2_1, 24); float dot_2_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2_1) : "f"(_add_632), "f"(_shfl_sync_637)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2_1) : "f"(_add_664), "f"(_shfl_sync_669)); float o_2_1[4]; - float _fma_639; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_639) : "f"(y_2_1[0]), "f"(dot_2_1), "f"(nz_1)); - float _sub_638; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_638) : "f"(gk_2[0]), "f"(_fma_639)); - o_2_1[0] = mul_f32_rn_noftz(rk_2[0], _sub_638); - float _fma_641; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_641) : "f"(y_2_1[1]), "f"(dot_2_1), "f"(nz_1)); - float _sub_640; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_640) : "f"(gk_2[1]), "f"(_fma_641)); - o_2_1[1] = mul_f32_rn_noftz(rk_2[0], _sub_640); - float _fma_643; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_643) : "f"(y_2_1[2]), "f"(dot_2_1), "f"(nz_1)); - float _sub_642; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_642) : "f"(gk_2[2]), "f"(_fma_643)); - o_2_1[2] = mul_f32_rn_noftz(rk_2[0], _sub_642); - float _fma_645; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_645) : "f"(y_2_1[3]), "f"(dot_2_1), "f"(nz_1)); - float _sub_644; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_644) : "f"(gk_2[3]), "f"(_fma_645)); - o_2_1[3] = mul_f32_rn_noftz(rk_2[0], _sub_644); + float _fma_671; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_671) : "f"(y_2_1[0]), "f"(dot_2_1), "f"(nz_1)); + float _sub_670; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_670) : "f"(gk_2[0]), "f"(_fma_671)); + o_2_1[0] = mul_f32_rn_noftz(rk_2[0], _sub_670); + float _fma_673; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_673) : "f"(y_2_1[1]), "f"(dot_2_1), "f"(nz_1)); + float _sub_672; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_672) : "f"(gk_2[1]), "f"(_fma_673)); + o_2_1[1] = mul_f32_rn_noftz(rk_2[0], _sub_672); + float _fma_675; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_675) : "f"(y_2_1[2]), "f"(dot_2_1), "f"(nz_1)); + float _sub_674; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_674) : "f"(gk_2[2]), "f"(_fma_675)); + o_2_1[2] = mul_f32_rn_noftz(rk_2[0], _sub_674); + float _fma_677; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_677) : "f"(y_2_1[3]), "f"(dot_2_1), "f"(nz_1)); + float _sub_676; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_676) : "f"(gk_2[3]), "f"(_fma_677)); + o_2_1[3] = mul_f32_rn_noftz(rk_2[0], _sub_676); unsigned int ob_2_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -5454,8 +5232,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_646 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_646), "f"(tot_1471) : "memory"); + uint32_t _addr_678 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_678), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5473,8 +5251,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other_0[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other_0[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi_1 = ((th == 1) ? tot_1471 : t_other_0[0]); - float t_lo_1 = ((th == 1) ? t_other_0[0] : tot_1471); + float t_hi_1 = ((th == 1) ? tot_1 : t_other_0[0]); + float t_lo_1 = ((th == 1) ? t_other_0[0] : tot_1); float offset_1 = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi_1); float sfx_1[8]; tmem_ld_x8(&sfx_1[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); @@ -5487,132 +5265,132 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_647 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_679 = make_float2(gacc[4], gacc[4]); float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_647), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_650; - asm("abs.f32 %0, %1;" : "=f"(_abs_650) : "f"(rp_z_7.x)); - float _neg_649; - asm("neg.f32 %0, %1;" : "=f"(_neg_649) : "f"(_abs_650)); - float _exp_648 = __nv_expf(_neg_649); - float _abs_653; - asm("abs.f32 %0, %1;" : "=f"(_abs_653) : "f"(rp_z_7.y)); - float _neg_652; - asm("neg.f32 %0, %1;" : "=f"(_neg_652) : "f"(_abs_653)); - float _exp_651 = __nv_expf(_neg_652); - float2 rp_e_7 = make_float2(_exp_648, _exp_651); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_679), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_682; + asm("abs.f32 %0, %1;" : "=f"(_abs_682) : "f"(rp_z_7.x)); + float _neg_681; + asm("neg.f32 %0, %1;" : "=f"(_neg_681) : "f"(_abs_682)); + float _exp_680 = __nv_expf(_neg_681); + float _abs_685; + asm("abs.f32 %0, %1;" : "=f"(_abs_685) : "f"(rp_z_7.y)); + float _neg_684; + asm("neg.f32 %0, %1;" : "=f"(_neg_684) : "f"(_abs_685)); + float _exp_683 = __nv_expf(_neg_684); + float2 rp_e_7 = make_float2(_exp_680, _exp_683); float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_654 = approx_rcp(rp_den_7.x); - float _rcp_655 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_656 = make_float2(_rcp_654, _rcp_655); + float _rcp_686 = approx_rcp(rp_den_7.x); + float _rcp_687 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_688 = make_float2(_rcp_686, _rcp_687); float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_656)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_688)); float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_657 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_689 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_657)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_689)); u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_658 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_690 = make_float2(gacc[4], gacc[4]); float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_658), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_661; - asm("abs.f32 %0, %1;" : "=f"(_abs_661) : "f"(rp_z_8.x)); - float _neg_660; - asm("neg.f32 %0, %1;" : "=f"(_neg_660) : "f"(_abs_661)); - float _exp_659 = __nv_expf(_neg_660); - float _abs_664; - asm("abs.f32 %0, %1;" : "=f"(_abs_664) : "f"(rp_z_8.y)); - float _neg_663; - asm("neg.f32 %0, %1;" : "=f"(_neg_663) : "f"(_abs_664)); - float _exp_662 = __nv_expf(_neg_663); - float2 rp_e_8 = make_float2(_exp_659, _exp_662); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_690), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_693; + asm("abs.f32 %0, %1;" : "=f"(_abs_693) : "f"(rp_z_8.x)); + float _neg_692; + asm("neg.f32 %0, %1;" : "=f"(_neg_692) : "f"(_abs_693)); + float _exp_691 = __nv_expf(_neg_692); + float _abs_696; + asm("abs.f32 %0, %1;" : "=f"(_abs_696) : "f"(rp_z_8.y)); + float _neg_695; + asm("neg.f32 %0, %1;" : "=f"(_neg_695) : "f"(_abs_696)); + float _exp_694 = __nv_expf(_neg_695); + float2 rp_e_8 = make_float2(_exp_691, _exp_694); float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_665 = approx_rcp(rp_den_8.x); - float _rcp_666 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_667 = make_float2(_rcp_665, _rcp_666); + float _rcp_697 = approx_rcp(rp_den_8.x); + float _rcp_698 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_699 = make_float2(_rcp_697, _rcp_698); float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_667)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_699)); float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_668 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_700 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_668)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_700)); u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_669 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_701 = make_float2(gacc[4], gacc[4]); float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_669), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_672; - asm("abs.f32 %0, %1;" : "=f"(_abs_672) : "f"(rp_z_9.x)); - float _neg_671; - asm("neg.f32 %0, %1;" : "=f"(_neg_671) : "f"(_abs_672)); - float _exp_670 = __nv_expf(_neg_671); - float _abs_675; - asm("abs.f32 %0, %1;" : "=f"(_abs_675) : "f"(rp_z_9.y)); - float _neg_674; - asm("neg.f32 %0, %1;" : "=f"(_neg_674) : "f"(_abs_675)); - float _exp_673 = __nv_expf(_neg_674); - float2 rp_e_9 = make_float2(_exp_670, _exp_673); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_701), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_704; + asm("abs.f32 %0, %1;" : "=f"(_abs_704) : "f"(rp_z_9.x)); + float _neg_703; + asm("neg.f32 %0, %1;" : "=f"(_neg_703) : "f"(_abs_704)); + float _exp_702 = __nv_expf(_neg_703); + float _abs_707; + asm("abs.f32 %0, %1;" : "=f"(_abs_707) : "f"(rp_z_9.y)); + float _neg_706; + asm("neg.f32 %0, %1;" : "=f"(_neg_706) : "f"(_abs_707)); + float _exp_705 = __nv_expf(_neg_706); + float2 rp_e_9 = make_float2(_exp_702, _exp_705); float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_676 = approx_rcp(rp_den_9.x); - float _rcp_677 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_678 = make_float2(_rcp_676, _rcp_677); + float _rcp_708 = approx_rcp(rp_den_9.x); + float _rcp_709 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_710 = make_float2(_rcp_708, _rcp_709); float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_678)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_710)); float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_679 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_711 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_679)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_711)); u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_680 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_712 = make_float2(gacc[4], gacc[4]); float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_680), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_683; - asm("abs.f32 %0, %1;" : "=f"(_abs_683) : "f"(rp_z_10.x)); - float _neg_682; - asm("neg.f32 %0, %1;" : "=f"(_neg_682) : "f"(_abs_683)); - float _exp_681 = __nv_expf(_neg_682); - float _abs_686; - asm("abs.f32 %0, %1;" : "=f"(_abs_686) : "f"(rp_z_10.y)); - float _neg_685; - asm("neg.f32 %0, %1;" : "=f"(_neg_685) : "f"(_abs_686)); - float _exp_684 = __nv_expf(_neg_685); - float2 rp_e_10 = make_float2(_exp_681, _exp_684); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_712), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_715; + asm("abs.f32 %0, %1;" : "=f"(_abs_715) : "f"(rp_z_10.x)); + float _neg_714; + asm("neg.f32 %0, %1;" : "=f"(_neg_714) : "f"(_abs_715)); + float _exp_713 = __nv_expf(_neg_714); + float _abs_718; + asm("abs.f32 %0, %1;" : "=f"(_abs_718) : "f"(rp_z_10.y)); + float _neg_717; + asm("neg.f32 %0, %1;" : "=f"(_neg_717) : "f"(_abs_718)); + float _exp_716 = __nv_expf(_neg_717); + float2 rp_e_10 = make_float2(_exp_713, _exp_716); float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_687 = approx_rcp(rp_den_10.x); - float _rcp_688 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_689 = make_float2(_rcp_687, _rcp_688); + float _rcp_719 = approx_rcp(rp_den_10.x); + float _rcp_720 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_721 = make_float2(_rcp_719, _rcp_720); float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_689)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_721)); float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_690 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_722 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_690)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_722)); u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -5627,54 +5405,54 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_691; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_691) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_691; + uint32_t _prmt_723; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_723) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_723; int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_692 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_692)); - *(reinterpret_cast(_gst_692) + (0)) = pq_8[0]; + auto _gst_724 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_724)); + *(reinterpret_cast(_gst_724) + (0)) = pq_8[0]; } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_693; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_693) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_693; + uint32_t _prmt_725; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_725) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_725; int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_694 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_694)); - *(reinterpret_cast(_gst_694) + (0)) = pq_9[0]; + auto _gst_726 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_726)); + *(reinterpret_cast(_gst_726) + (0)) = pq_9[0]; } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_695; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_695) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_695; + uint32_t _prmt_727; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_727) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_727; int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_696 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_696)); - *(reinterpret_cast(_gst_696) + (0)) = pq_10[0]; + auto _gst_728 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_728)); + *(reinterpret_cast(_gst_728) + (0)) = pq_10[0]; } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_697; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_697) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_697; + uint32_t _prmt_729; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_729) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_729; int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_698 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_698)); - *(reinterpret_cast(_gst_698) + (0)) = pq_11[0]; + auto _gst_730 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_730)); + *(reinterpret_cast(_gst_730) + (0)) = pq_11[0]; } } int t_a_16 = 16 + 8 * th; @@ -5739,47 +5517,47 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_699; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_699) : "f"(s_a_1), "f"(nm_t_3)); + float _sub_731; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_731) : "f"(s_a_1), "f"(nm_t_3)); float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_699), "f"(pa_1)); - float _sub_700; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_700) : "f"(pa_1), "f"(nm_t_3)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_731), "f"(pa_1)); + float _sub_732; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_732) : "f"(pa_1), "f"(nm_t_3)); float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_700), "f"(s_a_1)); - float _abs_702; - asm("abs.f32 %0, %1;" : "=f"(_abs_702) : "f"(s_a_1)); - float _abs_703; - asm("abs.f32 %0, %1;" : "=f"(_abs_703) : "f"(pa_1)); - uint32_t _setp_701_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_701_bits) : "f"(_abs_702), "f"(_abs_703)); - bool _setp_701 = _setp_701_bits != 0; - float nm_err_3 = ((_setp_701) ? nm_es_3 : nm_ex_3); - float _add_704; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_704) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_704; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_732), "f"(s_a_1)); + float _abs_734; + asm("abs.f32 %0, %1;" : "=f"(_abs_734) : "f"(s_a_1)); + float _abs_735; + asm("abs.f32 %0, %1;" : "=f"(_abs_735) : "f"(pa_1)); + uint32_t _setp_733_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_733_bits) : "f"(_abs_734), "f"(_abs_735)); + bool _setp_733 = _setp_733_bits != 0; + float nm_err_3 = ((_setp_733) ? nm_es_3 : nm_ex_3); + float _add_736; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_736) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_736; s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_705; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_705) : "f"(s_b_1), "f"(nm_t_4)); + float _sub_737; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_737) : "f"(s_b_1), "f"(nm_t_4)); float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_705), "f"(pb_1)); - float _sub_706; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_706) : "f"(pb_1), "f"(nm_t_4)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_737), "f"(pb_1)); + float _sub_738; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_738) : "f"(pb_1), "f"(nm_t_4)); float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_706), "f"(s_b_1)); - float _abs_708; - asm("abs.f32 %0, %1;" : "=f"(_abs_708) : "f"(s_b_1)); - float _abs_709; - asm("abs.f32 %0, %1;" : "=f"(_abs_709) : "f"(pb_1)); - uint32_t _setp_707_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_707_bits) : "f"(_abs_708), "f"(_abs_709)); - bool _setp_707 = _setp_707_bits != 0; - float nm_err_4 = ((_setp_707) ? nm_es_4 : nm_ex_4); - float _add_710; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_710) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_710; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_738), "f"(s_b_1)); + float _abs_740; + asm("abs.f32 %0, %1;" : "=f"(_abs_740) : "f"(s_b_1)); + float _abs_741; + asm("abs.f32 %0, %1;" : "=f"(_abs_741) : "f"(pb_1)); + uint32_t _setp_739_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_739_bits) : "f"(_abs_740), "f"(_abs_741)); + bool _setp_739 = _setp_739_bits != 0; + float nm_err_4 = ((_setp_739) ? nm_es_4 : nm_ex_4); + float _add_742; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_742) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_742; s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5787,8 +5565,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_711 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_711), "f"(off_acc[0]) : "memory"); + uint32_t _addr_743 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_743), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5812,362 +5590,166 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1514 = r_322[0]; - float gc_1515 = gn_1514 - 21.640425f; - int eb_1516 = 98304 + th * -32768; - int ew_1517 = eb_1516 >> 2; - int qbase_1518 = pool_f32_addr + (unsigned int)(eb_1516 + 12288 + dh_lane * 1024); - int sb8_1519 = 8 * th; - int gb8_1520 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1519 * 128); - int kb8_1521 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1519 * 128); - float dbp_1522[8]; - float _tmem_load_26[8]; - tmem_ld_x8(&_tmem_load_26[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float _tmem_load_27[8]; - tmem_ld_x8(&_tmem_load_27[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float _tmem_load_28[8]; - tmem_ld_x8(&_tmem_load_28[0], taddr + (unsigned int)row_base + (unsigned int)(8 * th)); - float btv_1523[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1523[0])), "=r"(*reinterpret_cast(&btv_1523[(0) + 1])), "=r"(*reinterpret_cast(&btv_1523[(0) + 2])), "=r"(*reinterpret_cast(&btv_1523[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1523[4])), "=r"(*reinterpret_cast(&btv_1523[(4) + 1])), "=r"(*reinterpret_cast(&btv_1523[(4) + 2])), "=r"(*reinterpret_cast(&btv_1523[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th) + 16))); + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_2 = r_280[0]; + float gc_2 = gn_2 - 21.640425f; + int eb_2 = 98304 + th * -32768; + int ew_5 = eb_2 >> 2; + int qbase_2 = pool_f32_addr + (unsigned int)(eb_2 + 12288 + dh_lane * 1024); + int sb8_2 = 8 * th; + int gb8_2 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_2 * 128); + int kb8_2 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_2 * 128); + int c_qa_1 = 8 * th; + int c_ka_1 = 8 * th; + int c_qd_1 = 128 + 8 * th; + int c_kd_1 = 144 + 8 * th; + int c_dk_1 = 8 * th; + int c_bp_1 = c_kd_1; + float s_run_2[1]; + s_run_2[0] = 0.0f; + #pragma unroll 1 + for (int k_2 = 0; k_2 < 8; k_2++) { + int j_5 = 7 - k_2; + float a_qd_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_1 + j_5))); + float a_kd_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_1 + j_5))); + float a_dk_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_1 + j_5))); + float btr_2[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_2[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_2 + j_5)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_2 = sb8_2 + j_5; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 * j_5 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_5 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 * j_5 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_5 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 * j_5 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_5 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_283[0] << 16); + float q_m_2 = ((t_2 >= clen) ? 0.0f : q_t_2); + float dqf_2 = pool_f32[ew_5 + 128 * j_5 + d_lane]; + float dkf_2 = pool_f32[ew_5 + (1024 + 128 * j_5) + d_lane]; + float dgf_2 = pool_f32[ew_5 + (2048 + 128 * j_5) + d_lane]; + float bt_t_2 = btr_2[0]; + float e_dg_2 = approx_exp2(g_t_2 - gc_2); + float dq2_2 = a_qd_2[0] * e_dg_2; + float dk2_2 = a_kd_2[0] * e_dg_2; + float dbv_2[1]; + dbv_2[0] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + a_dk_2[0]; + float qe_9 = q_m_2 * e_dg_2; + float kp_21 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, a_qd_2[0], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_2) * a_kd_2[0]); + float dg_o_2 = __fmaf_rn(q_m_2, dq2_2, __fmaf_rn(dk2b_2 - a_dk_2[0], k_t_2, cr_9)) + dgf_2; + s_run_2[0] = s_run_2[0] + dg_o_2; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_1 + j_5)), "f"(s_run_2[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_1 + j_5)), "f"(dbv_2[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(98304 - 32768 * th >> 2) + (128 * j_5 + d_lane)] = dq_o_2; + pool_f32[(98304 - 32768 * th >> 2) + (1024 + 128 * j_5 + d_lane)] = dk_o_2; + } + float tot_2 = s_run_2[0]; + float dbp_2[8]; + tmem_ld_x8(&dbp_2[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_1); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1524[1]; - s_run_1524[0] = 0.0f; - int t_16 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1517 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1517 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1517 + 2944 + d_lane]; - float bt_t_16 = btv_1523[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1514); - float e_dg_16 = approx_exp2(g_t_16 - gc_1515); - float dq2_16 = _tmem_load_26[7] * e_dg_16; - float dk2_16 = _tmem_load_27[7] * e_dg_16; - dbp_1522[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_28[7]; - float dg_o_16 = __fmaf_rn(q_t_1_24, dq2_16, (dk2b_16 - _tmem_load_28[7]) * k_t_16) + dgf_16; - s_run_1524[0] = s_run_1524[0] + dg_o_16; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_16; - pool_f32[(98304 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_16; - int t_17 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1517 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1517 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1517 + 2816 + d_lane]; - float bt_t_17 = btv_1523[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1514); - float e_dg_17 = approx_exp2(g_t_17 - gc_1515); - float dq2_17 = _tmem_load_26[6] * e_dg_17; - float dk2_17 = _tmem_load_27[6] * e_dg_17; - dbp_1522[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[6]; - float dg_o_17 = __fmaf_rn(q_t_1_25, dq2_17, (dk2b_17 - _tmem_load_28[6]) * k_t_17) + dgf_17; - s_run_1524[0] = s_run_1524[0] + dg_o_17; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_17; - pool_f32[(98304 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_17; - int t_18 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1517 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1517 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1517 + 2688 + d_lane]; - float bt_t_18 = btv_1523[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1514); - float e_dg_18 = approx_exp2(g_t_18 - gc_1515); - float dq2_18 = _tmem_load_26[5] * e_dg_18; - float dk2_18 = _tmem_load_27[5] * e_dg_18; - dbp_1522[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[5]; - float dg_o_18 = __fmaf_rn(q_t_1_26, dq2_18, (dk2b_18 - _tmem_load_28[5]) * k_t_18) + dgf_18; - s_run_1524[0] = s_run_1524[0] + dg_o_18; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_18; - pool_f32[(98304 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_18; - int t_19 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1517 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1517 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1517 + 2560 + d_lane]; - float bt_t_19 = btv_1523[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1514); - float e_dg_19 = approx_exp2(g_t_19 - gc_1515); - float dq2_19 = _tmem_load_26[4] * e_dg_19; - float dk2_19 = _tmem_load_27[4] * e_dg_19; - dbp_1522[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[4]; - float dg_o_19 = __fmaf_rn(q_t_1_27, dq2_19, (dk2b_19 - _tmem_load_28[4]) * k_t_19) + dgf_19; - s_run_1524[0] = s_run_1524[0] + dg_o_19; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_19; - pool_f32[(98304 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_19; - int t_20 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1517 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1517 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1517 + 2432 + d_lane]; - float bt_t_20 = btv_1523[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1514); - float e_dg_20 = approx_exp2(g_t_20 - gc_1515); - float dq2_20 = _tmem_load_26[3] * e_dg_20; - float dk2_20 = _tmem_load_27[3] * e_dg_20; - dbp_1522[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[3]; - float dg_o_20 = __fmaf_rn(q_t_1_28, dq2_20, (dk2b_20 - _tmem_load_28[3]) * k_t_20) + dgf_20; - s_run_1524[0] = s_run_1524[0] + dg_o_20; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_20; - pool_f32[(98304 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_20; - int t_21 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1517 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1517 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1517 + 2304 + d_lane]; - float bt_t_21 = btv_1523[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1514); - float e_dg_21 = approx_exp2(g_t_21 - gc_1515); - float dq2_21 = _tmem_load_26[2] * e_dg_21; - float dk2_21 = _tmem_load_27[2] * e_dg_21; - dbp_1522[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[2]; - float dg_o_21 = __fmaf_rn(q_t_1_29, dq2_21, (dk2b_21 - _tmem_load_28[2]) * k_t_21) + dgf_21; - s_run_1524[0] = s_run_1524[0] + dg_o_21; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_21; - pool_f32[(98304 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_21; - int t_22 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1517 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1517 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1517 + 2176 + d_lane]; - float bt_t_22 = btv_1523[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1514); - float e_dg_22 = approx_exp2(g_t_22 - gc_1515); - float dq2_22 = _tmem_load_26[1] * e_dg_22; - float dk2_22 = _tmem_load_27[1] * e_dg_22; - dbp_1522[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[1]; - float dg_o_22 = __fmaf_rn(q_t_1_30, dq2_22, (dk2b_22 - _tmem_load_28[1]) * k_t_22) + dgf_22; - s_run_1524[0] = s_run_1524[0] + dg_o_22; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_22; - pool_f32[(98304 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_22; - int t_23 = 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1517 + d_lane]; - float dkf_23 = pool_f32[ew_1517 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1517 + 2048 + d_lane]; - float bt_t_23 = btv_1523[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1514); - float e_dg_23 = approx_exp2(g_t_23 - gc_1515); - float dq2_23 = _tmem_load_26[0] * e_dg_23; - float dk2_23 = _tmem_load_27[0] * e_dg_23; - dbp_1522[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[0]; - float dg_o_23 = __fmaf_rn(q_t_1_31, dq2_23, (dk2b_23 - _tmem_load_28[0]) * k_t_23) + dgf_23; - s_run_1524[0] = s_run_1524[0] + dg_o_23; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + d_lane] = dq_o_23; - pool_f32[(98304 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_23; - float tot_1693 = s_run_1524[0]; - int b4_1694 = lane_1 >> 4 & 1; - int b3_1695 = lane_1 >> 3 & 1; - int b2_1696 = lane_1 >> 2 & 1; - float v4_1697[4]; - float send_12 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_712 = __shfl_xor_sync(0xFFFFFFFF, send_12, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_712; - float send_13 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_713 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_713; - float send_14 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_714 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_714; - float send_15 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_715 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_715; - float v2_1702[2]; - float send_16 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_716 = __shfl_xor_sync(0xFFFFFFFF, send_16, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_716; - float send_17 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_717 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_717; - float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_718 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_718; - float _shfl_xor_sync_719 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_719; - float _shfl_xor_sync_720 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_720; - int ew_3 = 98304 - 32768 * th >> 2; + int b4_2 = lane_1 >> 4 & 1; + int b3_2 = lane_1 >> 3 & 1; + int b2_2 = lane_1 >> 2 & 1; + float v4_2[4]; + float send_8 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); + float _shfl_xor_sync_744 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_744; + float send_9 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); + float _shfl_xor_sync_745 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_745; + float send_10 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); + float _shfl_xor_sync_746 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_746; + float send_11 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); + float _shfl_xor_sync_747 = __shfl_xor_sync(0xFFFFFFFF, send_11, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_747; + float v2_17[2]; + float send2_4 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); + float _shfl_xor_sync_748 = __shfl_xor_sync(0xFFFFFFFF, send2_4, 8); + v2_17[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_748; + float send2_5 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); + float _shfl_xor_sync_749 = __shfl_xor_sync(0xFFFFFFFF, send2_5, 8); + v2_17[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_749; + float send1_1 = ((b2_2 == 0) ? v2_17[1] : v2_17[0]); + float _shfl_xor_sync_750 = __shfl_xor_sync(0xFFFFFFFF, send1_1, 4); + float v1_2 = ((b2_2 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_750; + float _shfl_xor_sync_751 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_751; + float _shfl_xor_sync_752 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_752; + int ew_6 = 98304 - 32768 * th >> 2; float nz_2 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); asm volatile("barrier.sync 1, 256;" ::: "memory"); #pragma unroll 1 for (int i_3 = 0; i_3 < 2; i_3++) { - int j_3 = qd + 4 * i_3; - bool live_3 = 8 * th + j_3 < clen; - int _min_721 = ((8 * th + j_3) < (clen - 1) ? (8 * th + j_3) : (clen - 1)); - long long row_3 = ext0 + (long long)_min_721; + int j_6 = qd + 4 * i_3; + bool live_3 = 8 * th + j_6 < clen; + int _min_753 = ((8 * th + j_6) < (clen - 1) ? (8 * th + j_6) : (clen - 1)); + long long row_3 = ext0 + (long long)_min_753; float gq_3[4]; float gk_3[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gq_3[0])), "=r"(*reinterpret_cast(&gq_3[(0) + 1])), "=r"(*reinterpret_cast(&gq_3[(0) + 2])), "=r"(*reinterpret_cast(&gq_3[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_3 + 128 * j_3 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_6 + 128 * j_6 + 4 * lane_1)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gk_3[0])), "=r"(*reinterpret_cast(&gk_3[(0) + 1])), "=r"(*reinterpret_cast(&gk_3[(0) + 2])), "=r"(*reinterpret_cast(&gk_3[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_3 + 1024 + 128 * j_3 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_6 + 1024 + 128 * j_6 + 4 * lane_1)))); float xq_3[4]; float xk_3[4]; float rq_3[1]; float rk_3[1]; { - uint2 _vld_722; - _vld_722 = *reinterpret_cast(q_raw + (row_3 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_722 = reinterpret_cast(&_vld_722); + uint2 _vld_754; + _vld_754 = *reinterpret_cast(q_raw + (row_3 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_754 = reinterpret_cast(&_vld_754); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xq_3[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_722[_pair]) << 16); - (&xq_3[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_722[_pair]) & 0xffff0000u); + (&xq_3[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_754[_pair]) << 16); + (&xq_3[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_754[_pair]) & 0xffff0000u); } } { rq_3[0] = *reinterpret_cast(qk_rstd + ((0 + row_3) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_723; - _vld_723 = *reinterpret_cast(k_raw + (row_3 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_723 = reinterpret_cast(&_vld_723); + uint2 _vld_755; + _vld_755 = *reinterpret_cast(k_raw + (row_3 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_755 = reinterpret_cast(&_vld_755); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xk_3[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_723[_pair]) << 16); - (&xk_3[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_723[_pair]) & 0xffff0000u); + (&xk_3[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_755[_pair]) << 16); + (&xk_3[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_755[_pair]) & 0xffff0000u); } } { @@ -6177,106 +5759,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_3[4]; float p_3[4]; y_3[0] = mul_f32_rn_noftz(xq_3[0], rq_3[0]); - float _fma_724; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_724) : "f"(gq_3[0]), "f"(y_3[0]), "f"(nz_2)); - p_3[0] = _fma_724; + float _fma_756; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_756) : "f"(gq_3[0]), "f"(y_3[0]), "f"(nz_2)); + p_3[0] = _fma_756; y_3[1] = mul_f32_rn_noftz(xq_3[1], rq_3[0]); - float _fma_725; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_725) : "f"(gq_3[1]), "f"(y_3[1]), "f"(nz_2)); - p_3[1] = _fma_725; + float _fma_757; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_757) : "f"(gq_3[1]), "f"(y_3[1]), "f"(nz_2)); + p_3[1] = _fma_757; y_3[2] = mul_f32_rn_noftz(xq_3[2], rq_3[0]); - float _fma_726; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_726) : "f"(gq_3[2]), "f"(y_3[2]), "f"(nz_2)); - p_3[2] = _fma_726; + float _fma_758; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_758) : "f"(gq_3[2]), "f"(y_3[2]), "f"(nz_2)); + p_3[2] = _fma_758; y_3[3] = mul_f32_rn_noftz(xq_3[3], rq_3[0]); - float _fma_727; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_727) : "f"(gq_3[3]), "f"(y_3[3]), "f"(nz_2)); - p_3[3] = _fma_727; - float _shfl_xor_sync_729 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 4); - float _add_728; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_728) : "f"(p_3[0]), "f"(_shfl_xor_sync_729)); - p_3[0] = _add_728; - float _shfl_xor_sync_731 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 4); - float _add_730; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_730) : "f"(p_3[1]), "f"(_shfl_xor_sync_731)); - p_3[1] = _add_730; - float _shfl_xor_sync_733 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 4); - float _add_732; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_732) : "f"(p_3[2]), "f"(_shfl_xor_sync_733)); - p_3[2] = _add_732; - float _shfl_xor_sync_735 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 4); - float _add_734; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_734) : "f"(p_3[3]), "f"(_shfl_xor_sync_735)); - p_3[3] = _add_734; - float _shfl_xor_sync_737 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 2); - float _add_736; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_736) : "f"(p_3[0]), "f"(_shfl_xor_sync_737)); - p_3[0] = _add_736; - float _shfl_xor_sync_739 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 2); - float _add_738; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_738) : "f"(p_3[1]), "f"(_shfl_xor_sync_739)); - p_3[1] = _add_738; - float _shfl_xor_sync_741 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 2); - float _add_740; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_740) : "f"(p_3[2]), "f"(_shfl_xor_sync_741)); - p_3[2] = _add_740; - float _shfl_xor_sync_743 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 2); - float _add_742; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_742) : "f"(p_3[3]), "f"(_shfl_xor_sync_743)); - p_3[3] = _add_742; - float _shfl_xor_sync_745 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 1); - float _add_744; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_744) : "f"(p_3[0]), "f"(_shfl_xor_sync_745)); - p_3[0] = _add_744; - float _shfl_xor_sync_747 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 1); - float _add_746; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_746) : "f"(p_3[1]), "f"(_shfl_xor_sync_747)); - p_3[1] = _add_746; - float _shfl_xor_sync_749 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 1); - float _add_748; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_748) : "f"(p_3[2]), "f"(_shfl_xor_sync_749)); - p_3[2] = _add_748; - float _shfl_xor_sync_751 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 1); - float _add_750; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_750) : "f"(p_3[3]), "f"(_shfl_xor_sync_751)); - p_3[3] = _add_750; - float _add_752; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_752) : "f"(p_3[0]), "f"(p_3[2])); - float _add_753; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_753) : "f"(p_3[1]), "f"(p_3[3])); + float _fma_759; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_759) : "f"(gq_3[3]), "f"(y_3[3]), "f"(nz_2)); + p_3[3] = _fma_759; + float _shfl_xor_sync_761 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 4); + float _add_760; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_760) : "f"(p_3[0]), "f"(_shfl_xor_sync_761)); + p_3[0] = _add_760; + float _shfl_xor_sync_763 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 4); + float _add_762; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_762) : "f"(p_3[1]), "f"(_shfl_xor_sync_763)); + p_3[1] = _add_762; + float _shfl_xor_sync_765 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 4); + float _add_764; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_764) : "f"(p_3[2]), "f"(_shfl_xor_sync_765)); + p_3[2] = _add_764; + float _shfl_xor_sync_767 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 4); + float _add_766; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_766) : "f"(p_3[3]), "f"(_shfl_xor_sync_767)); + p_3[3] = _add_766; + float _shfl_xor_sync_769 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 2); + float _add_768; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_768) : "f"(p_3[0]), "f"(_shfl_xor_sync_769)); + p_3[0] = _add_768; + float _shfl_xor_sync_771 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 2); + float _add_770; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_770) : "f"(p_3[1]), "f"(_shfl_xor_sync_771)); + p_3[1] = _add_770; + float _shfl_xor_sync_773 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 2); + float _add_772; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_772) : "f"(p_3[2]), "f"(_shfl_xor_sync_773)); + p_3[2] = _add_772; + float _shfl_xor_sync_775 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 2); + float _add_774; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_774) : "f"(p_3[3]), "f"(_shfl_xor_sync_775)); + p_3[3] = _add_774; + float _shfl_xor_sync_777 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 1); + float _add_776; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_776) : "f"(p_3[0]), "f"(_shfl_xor_sync_777)); + p_3[0] = _add_776; + float _shfl_xor_sync_779 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 1); + float _add_778; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_778) : "f"(p_3[1]), "f"(_shfl_xor_sync_779)); + p_3[1] = _add_778; + float _shfl_xor_sync_781 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 1); + float _add_780; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_780) : "f"(p_3[2]), "f"(_shfl_xor_sync_781)); + p_3[2] = _add_780; + float _shfl_xor_sync_783 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 1); + float _add_782; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_782) : "f"(p_3[3]), "f"(_shfl_xor_sync_783)); + p_3[3] = _add_782; + float _add_784; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_784) : "f"(p_3[0]), "f"(p_3[2])); + float _add_785; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_785) : "f"(p_3[1]), "f"(p_3[3])); float sq_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3) : "f"(_add_752), "f"(_add_753)); - float _shfl_sync_756 = __shfl_sync(0xFFFFFFFF, sq_3, 0); - float _shfl_sync_757 = __shfl_sync(0xFFFFFFFF, sq_3, 8); - float _add_755; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_755) : "f"(_shfl_sync_756), "f"(_shfl_sync_757)); - float _shfl_sync_758 = __shfl_sync(0xFFFFFFFF, sq_3, 16); - float _add_754; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_754) : "f"(_add_755), "f"(_shfl_sync_758)); - float _shfl_sync_759 = __shfl_sync(0xFFFFFFFF, sq_3, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3) : "f"(_add_784), "f"(_add_785)); + float _shfl_sync_788 = __shfl_sync(0xFFFFFFFF, sq_3, 0); + float _shfl_sync_789 = __shfl_sync(0xFFFFFFFF, sq_3, 8); + float _add_787; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_787) : "f"(_shfl_sync_788), "f"(_shfl_sync_789)); + float _shfl_sync_790 = __shfl_sync(0xFFFFFFFF, sq_3, 16); + float _add_786; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_786) : "f"(_add_787), "f"(_shfl_sync_790)); + float _shfl_sync_791 = __shfl_sync(0xFFFFFFFF, sq_3, 24); float dot_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3) : "f"(_add_754), "f"(_shfl_sync_759)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3) : "f"(_add_786), "f"(_shfl_sync_791)); float o_3[4]; - float _fma_761; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_761) : "f"(y_3[0]), "f"(dot_3), "f"(nz_2)); - float _sub_760; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_760) : "f"(gq_3[0]), "f"(_fma_761)); - o_3[0] = mul_f32_rn_noftz(rq_3[0], _sub_760); - float _fma_763; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_763) : "f"(y_3[1]), "f"(dot_3), "f"(nz_2)); - float _sub_762; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_762) : "f"(gq_3[1]), "f"(_fma_763)); - o_3[1] = mul_f32_rn_noftz(rq_3[0], _sub_762); - float _fma_765; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_765) : "f"(y_3[2]), "f"(dot_3), "f"(nz_2)); - float _sub_764; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_764) : "f"(gq_3[2]), "f"(_fma_765)); - o_3[2] = mul_f32_rn_noftz(rq_3[0], _sub_764); - float _fma_767; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_767) : "f"(y_3[3]), "f"(dot_3), "f"(nz_2)); - float _sub_766; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_766) : "f"(gq_3[3]), "f"(_fma_767)); - o_3[3] = mul_f32_rn_noftz(rq_3[0], _sub_766); + float _fma_793; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_793) : "f"(y_3[0]), "f"(dot_3), "f"(nz_2)); + float _sub_792; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_792) : "f"(gq_3[0]), "f"(_fma_793)); + o_3[0] = mul_f32_rn_noftz(rq_3[0], _sub_792); + float _fma_795; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_795) : "f"(y_3[1]), "f"(dot_3), "f"(nz_2)); + float _sub_794; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_794) : "f"(gq_3[1]), "f"(_fma_795)); + o_3[1] = mul_f32_rn_noftz(rq_3[0], _sub_794); + float _fma_797; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_797) : "f"(y_3[2]), "f"(dot_3), "f"(nz_2)); + float _sub_796; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_796) : "f"(gq_3[2]), "f"(_fma_797)); + o_3[2] = mul_f32_rn_noftz(rq_3[0], _sub_796); + float _fma_799; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_799) : "f"(y_3[3]), "f"(dot_3), "f"(nz_2)); + float _sub_798; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_798) : "f"(gq_3[3]), "f"(_fma_799)); + o_3[3] = mul_f32_rn_noftz(rq_3[0], _sub_798); unsigned int ob_3[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -6287,106 +5869,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_3_1[4]; float p_3_1[4]; y_3_1[0] = mul_f32_rn_noftz(xk_3[0], rk_3[0]); - float _fma_768; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_768) : "f"(gk_3[0]), "f"(y_3_1[0]), "f"(nz_2)); - p_3_1[0] = _fma_768; + float _fma_800; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_800) : "f"(gk_3[0]), "f"(y_3_1[0]), "f"(nz_2)); + p_3_1[0] = _fma_800; y_3_1[1] = mul_f32_rn_noftz(xk_3[1], rk_3[0]); - float _fma_769; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_769) : "f"(gk_3[1]), "f"(y_3_1[1]), "f"(nz_2)); - p_3_1[1] = _fma_769; + float _fma_801; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_801) : "f"(gk_3[1]), "f"(y_3_1[1]), "f"(nz_2)); + p_3_1[1] = _fma_801; y_3_1[2] = mul_f32_rn_noftz(xk_3[2], rk_3[0]); - float _fma_770; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_770) : "f"(gk_3[2]), "f"(y_3_1[2]), "f"(nz_2)); - p_3_1[2] = _fma_770; + float _fma_802; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_802) : "f"(gk_3[2]), "f"(y_3_1[2]), "f"(nz_2)); + p_3_1[2] = _fma_802; y_3_1[3] = mul_f32_rn_noftz(xk_3[3], rk_3[0]); - float _fma_771; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_771) : "f"(gk_3[3]), "f"(y_3_1[3]), "f"(nz_2)); - p_3_1[3] = _fma_771; - float _shfl_xor_sync_773 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 4); - float _add_772; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_772) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_773)); - p_3_1[0] = _add_772; - float _shfl_xor_sync_775 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 4); - float _add_774; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_774) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_775)); - p_3_1[1] = _add_774; - float _shfl_xor_sync_777 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 4); - float _add_776; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_776) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_777)); - p_3_1[2] = _add_776; - float _shfl_xor_sync_779 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 4); - float _add_778; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_778) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_779)); - p_3_1[3] = _add_778; - float _shfl_xor_sync_781 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 2); - float _add_780; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_780) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_781)); - p_3_1[0] = _add_780; - float _shfl_xor_sync_783 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 2); - float _add_782; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_782) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_783)); - p_3_1[1] = _add_782; - float _shfl_xor_sync_785 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 2); - float _add_784; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_784) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_785)); - p_3_1[2] = _add_784; - float _shfl_xor_sync_787 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 2); - float _add_786; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_786) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_787)); - p_3_1[3] = _add_786; - float _shfl_xor_sync_789 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 1); - float _add_788; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_788) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_789)); - p_3_1[0] = _add_788; - float _shfl_xor_sync_791 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 1); - float _add_790; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_790) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_791)); - p_3_1[1] = _add_790; - float _shfl_xor_sync_793 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 1); - float _add_792; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_792) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_793)); - p_3_1[2] = _add_792; - float _shfl_xor_sync_795 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 1); - float _add_794; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_794) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_795)); - p_3_1[3] = _add_794; - float _add_796; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_796) : "f"(p_3_1[0]), "f"(p_3_1[2])); - float _add_797; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_797) : "f"(p_3_1[1]), "f"(p_3_1[3])); + float _fma_803; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_803) : "f"(gk_3[3]), "f"(y_3_1[3]), "f"(nz_2)); + p_3_1[3] = _fma_803; + float _shfl_xor_sync_805 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 4); + float _add_804; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_804) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_805)); + p_3_1[0] = _add_804; + float _shfl_xor_sync_807 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 4); + float _add_806; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_806) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_807)); + p_3_1[1] = _add_806; + float _shfl_xor_sync_809 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 4); + float _add_808; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_808) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_809)); + p_3_1[2] = _add_808; + float _shfl_xor_sync_811 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 4); + float _add_810; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_810) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_811)); + p_3_1[3] = _add_810; + float _shfl_xor_sync_813 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 2); + float _add_812; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_812) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_813)); + p_3_1[0] = _add_812; + float _shfl_xor_sync_815 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 2); + float _add_814; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_814) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_815)); + p_3_1[1] = _add_814; + float _shfl_xor_sync_817 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 2); + float _add_816; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_816) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_817)); + p_3_1[2] = _add_816; + float _shfl_xor_sync_819 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 2); + float _add_818; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_818) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_819)); + p_3_1[3] = _add_818; + float _shfl_xor_sync_821 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 1); + float _add_820; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_820) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_821)); + p_3_1[0] = _add_820; + float _shfl_xor_sync_823 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 1); + float _add_822; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_822) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_823)); + p_3_1[1] = _add_822; + float _shfl_xor_sync_825 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 1); + float _add_824; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_824) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_825)); + p_3_1[2] = _add_824; + float _shfl_xor_sync_827 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 1); + float _add_826; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_826) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_827)); + p_3_1[3] = _add_826; + float _add_828; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_828) : "f"(p_3_1[0]), "f"(p_3_1[2])); + float _add_829; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_829) : "f"(p_3_1[1]), "f"(p_3_1[3])); float sq_3_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3_1) : "f"(_add_796), "f"(_add_797)); - float _shfl_sync_800 = __shfl_sync(0xFFFFFFFF, sq_3_1, 0); - float _shfl_sync_801 = __shfl_sync(0xFFFFFFFF, sq_3_1, 8); - float _add_799; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_799) : "f"(_shfl_sync_800), "f"(_shfl_sync_801)); - float _shfl_sync_802 = __shfl_sync(0xFFFFFFFF, sq_3_1, 16); - float _add_798; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_798) : "f"(_add_799), "f"(_shfl_sync_802)); - float _shfl_sync_803 = __shfl_sync(0xFFFFFFFF, sq_3_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3_1) : "f"(_add_828), "f"(_add_829)); + float _shfl_sync_832 = __shfl_sync(0xFFFFFFFF, sq_3_1, 0); + float _shfl_sync_833 = __shfl_sync(0xFFFFFFFF, sq_3_1, 8); + float _add_831; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_831) : "f"(_shfl_sync_832), "f"(_shfl_sync_833)); + float _shfl_sync_834 = __shfl_sync(0xFFFFFFFF, sq_3_1, 16); + float _add_830; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_830) : "f"(_add_831), "f"(_shfl_sync_834)); + float _shfl_sync_835 = __shfl_sync(0xFFFFFFFF, sq_3_1, 24); float dot_3_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3_1) : "f"(_add_798), "f"(_shfl_sync_803)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3_1) : "f"(_add_830), "f"(_shfl_sync_835)); float o_3_1[4]; - float _fma_805; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_805) : "f"(y_3_1[0]), "f"(dot_3_1), "f"(nz_2)); - float _sub_804; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_804) : "f"(gk_3[0]), "f"(_fma_805)); - o_3_1[0] = mul_f32_rn_noftz(rk_3[0], _sub_804); - float _fma_807; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_807) : "f"(y_3_1[1]), "f"(dot_3_1), "f"(nz_2)); - float _sub_806; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_806) : "f"(gk_3[1]), "f"(_fma_807)); - o_3_1[1] = mul_f32_rn_noftz(rk_3[0], _sub_806); - float _fma_809; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_809) : "f"(y_3_1[2]), "f"(dot_3_1), "f"(nz_2)); - float _sub_808; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_808) : "f"(gk_3[2]), "f"(_fma_809)); - o_3_1[2] = mul_f32_rn_noftz(rk_3[0], _sub_808); - float _fma_811; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_811) : "f"(y_3_1[3]), "f"(dot_3_1), "f"(nz_2)); - float _sub_810; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_810) : "f"(gk_3[3]), "f"(_fma_811)); - o_3_1[3] = mul_f32_rn_noftz(rk_3[0], _sub_810); + float _fma_837; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_837) : "f"(y_3_1[0]), "f"(dot_3_1), "f"(nz_2)); + float _sub_836; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_836) : "f"(gk_3[0]), "f"(_fma_837)); + o_3_1[0] = mul_f32_rn_noftz(rk_3[0], _sub_836); + float _fma_839; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_839) : "f"(y_3_1[1]), "f"(dot_3_1), "f"(nz_2)); + float _sub_838; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_838) : "f"(gk_3[1]), "f"(_fma_839)); + o_3_1[1] = mul_f32_rn_noftz(rk_3[0], _sub_838); + float _fma_841; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_841) : "f"(y_3_1[2]), "f"(dot_3_1), "f"(nz_2)); + float _sub_840; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_840) : "f"(gk_3[2]), "f"(_fma_841)); + o_3_1[2] = mul_f32_rn_noftz(rk_3[0], _sub_840); + float _fma_843; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_843) : "f"(y_3_1[3]), "f"(dot_3_1), "f"(nz_2)); + float _sub_842; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_842) : "f"(gk_3[3]), "f"(_fma_843)); + o_3_1[3] = mul_f32_rn_noftz(rk_3[0], _sub_842); unsigned int ob_3_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -6399,8 +5981,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_812 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_812), "f"(tot_1693) : "memory"); + uint32_t _addr_844 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_844), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -6418,8 +6000,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other_1[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other_1[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi_2 = ((th == 1) ? tot_1693 : t_other_1[0]); - float t_lo_2 = ((th == 1) ? t_other_1[0] : tot_1693); + float t_hi_2 = ((th == 1) ? tot_2 : t_other_1[0]); + float t_lo_2 = ((th == 1) ? t_other_1[0] : tot_2); float offset_2 = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi_2); float sfx_2[8]; tmem_ld_x8(&sfx_2[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); @@ -6432,132 +6014,132 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_813 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_845 = make_float2(gacc[4], gacc[4]); float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_813), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_816; - asm("abs.f32 %0, %1;" : "=f"(_abs_816) : "f"(rp_z_11.x)); - float _neg_815; - asm("neg.f32 %0, %1;" : "=f"(_neg_815) : "f"(_abs_816)); - float _exp_814 = __nv_expf(_neg_815); - float _abs_819; - asm("abs.f32 %0, %1;" : "=f"(_abs_819) : "f"(rp_z_11.y)); - float _neg_818; - asm("neg.f32 %0, %1;" : "=f"(_neg_818) : "f"(_abs_819)); - float _exp_817 = __nv_expf(_neg_818); - float2 rp_e_11 = make_float2(_exp_814, _exp_817); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_845), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_848; + asm("abs.f32 %0, %1;" : "=f"(_abs_848) : "f"(rp_z_11.x)); + float _neg_847; + asm("neg.f32 %0, %1;" : "=f"(_neg_847) : "f"(_abs_848)); + float _exp_846 = __nv_expf(_neg_847); + float _abs_851; + asm("abs.f32 %0, %1;" : "=f"(_abs_851) : "f"(rp_z_11.y)); + float _neg_850; + asm("neg.f32 %0, %1;" : "=f"(_neg_850) : "f"(_abs_851)); + float _exp_849 = __nv_expf(_neg_850); + float2 rp_e_11 = make_float2(_exp_846, _exp_849); float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_820 = approx_rcp(rp_den_11.x); - float _rcp_821 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_822 = make_float2(_rcp_820, _rcp_821); + float _rcp_852 = approx_rcp(rp_den_11.x); + float _rcp_853 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_854 = make_float2(_rcp_852, _rcp_853); float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_822)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_854)); float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_823 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_855 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_823)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_855)); u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_824 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_856 = make_float2(gacc[4], gacc[4]); float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_824), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_827; - asm("abs.f32 %0, %1;" : "=f"(_abs_827) : "f"(rp_z_12.x)); - float _neg_826; - asm("neg.f32 %0, %1;" : "=f"(_neg_826) : "f"(_abs_827)); - float _exp_825 = __nv_expf(_neg_826); - float _abs_830; - asm("abs.f32 %0, %1;" : "=f"(_abs_830) : "f"(rp_z_12.y)); - float _neg_829; - asm("neg.f32 %0, %1;" : "=f"(_neg_829) : "f"(_abs_830)); - float _exp_828 = __nv_expf(_neg_829); - float2 rp_e_12 = make_float2(_exp_825, _exp_828); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_856), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_859; + asm("abs.f32 %0, %1;" : "=f"(_abs_859) : "f"(rp_z_12.x)); + float _neg_858; + asm("neg.f32 %0, %1;" : "=f"(_neg_858) : "f"(_abs_859)); + float _exp_857 = __nv_expf(_neg_858); + float _abs_862; + asm("abs.f32 %0, %1;" : "=f"(_abs_862) : "f"(rp_z_12.y)); + float _neg_861; + asm("neg.f32 %0, %1;" : "=f"(_neg_861) : "f"(_abs_862)); + float _exp_860 = __nv_expf(_neg_861); + float2 rp_e_12 = make_float2(_exp_857, _exp_860); float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_831 = approx_rcp(rp_den_12.x); - float _rcp_832 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_833 = make_float2(_rcp_831, _rcp_832); + float _rcp_863 = approx_rcp(rp_den_12.x); + float _rcp_864 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_865 = make_float2(_rcp_863, _rcp_864); float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_833)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_865)); float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_834 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_866 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_834)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_866)); u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_835 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_867 = make_float2(gacc[4], gacc[4]); float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_835), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_838; - asm("abs.f32 %0, %1;" : "=f"(_abs_838) : "f"(rp_z_13.x)); - float _neg_837; - asm("neg.f32 %0, %1;" : "=f"(_neg_837) : "f"(_abs_838)); - float _exp_836 = __nv_expf(_neg_837); - float _abs_841; - asm("abs.f32 %0, %1;" : "=f"(_abs_841) : "f"(rp_z_13.y)); - float _neg_840; - asm("neg.f32 %0, %1;" : "=f"(_neg_840) : "f"(_abs_841)); - float _exp_839 = __nv_expf(_neg_840); - float2 rp_e_13 = make_float2(_exp_836, _exp_839); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_867), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_870; + asm("abs.f32 %0, %1;" : "=f"(_abs_870) : "f"(rp_z_13.x)); + float _neg_869; + asm("neg.f32 %0, %1;" : "=f"(_neg_869) : "f"(_abs_870)); + float _exp_868 = __nv_expf(_neg_869); + float _abs_873; + asm("abs.f32 %0, %1;" : "=f"(_abs_873) : "f"(rp_z_13.y)); + float _neg_872; + asm("neg.f32 %0, %1;" : "=f"(_neg_872) : "f"(_abs_873)); + float _exp_871 = __nv_expf(_neg_872); + float2 rp_e_13 = make_float2(_exp_868, _exp_871); float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_842 = approx_rcp(rp_den_13.x); - float _rcp_843 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_844 = make_float2(_rcp_842, _rcp_843); + float _rcp_874 = approx_rcp(rp_den_13.x); + float _rcp_875 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_876 = make_float2(_rcp_874, _rcp_875); float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_844)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_876)); float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_845 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_877 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_845)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_877)); u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_846 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_878 = make_float2(gacc[4], gacc[4]); float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_846), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_849; - asm("abs.f32 %0, %1;" : "=f"(_abs_849) : "f"(rp_z_14.x)); - float _neg_848; - asm("neg.f32 %0, %1;" : "=f"(_neg_848) : "f"(_abs_849)); - float _exp_847 = __nv_expf(_neg_848); - float _abs_852; - asm("abs.f32 %0, %1;" : "=f"(_abs_852) : "f"(rp_z_14.y)); - float _neg_851; - asm("neg.f32 %0, %1;" : "=f"(_neg_851) : "f"(_abs_852)); - float _exp_850 = __nv_expf(_neg_851); - float2 rp_e_14 = make_float2(_exp_847, _exp_850); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_878), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_881; + asm("abs.f32 %0, %1;" : "=f"(_abs_881) : "f"(rp_z_14.x)); + float _neg_880; + asm("neg.f32 %0, %1;" : "=f"(_neg_880) : "f"(_abs_881)); + float _exp_879 = __nv_expf(_neg_880); + float _abs_884; + asm("abs.f32 %0, %1;" : "=f"(_abs_884) : "f"(rp_z_14.y)); + float _neg_883; + asm("neg.f32 %0, %1;" : "=f"(_neg_883) : "f"(_abs_884)); + float _exp_882 = __nv_expf(_neg_883); + float2 rp_e_14 = make_float2(_exp_879, _exp_882); float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_853 = approx_rcp(rp_den_14.x); - float _rcp_854 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_855 = make_float2(_rcp_853, _rcp_854); + float _rcp_885 = approx_rcp(rp_den_14.x); + float _rcp_886 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_887 = make_float2(_rcp_885, _rcp_886); float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_855)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_887)); float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_856 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_888 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_856)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_888)); u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -6572,54 +6154,54 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_857; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_857) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_857; + uint32_t _prmt_889; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_889) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_889; int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_858 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_858)); - *(reinterpret_cast(_gst_858) + (0)) = pq_12[0]; + auto _gst_890 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_890)); + *(reinterpret_cast(_gst_890) + (0)) = pq_12[0]; } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_859; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_859) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_859; + uint32_t _prmt_891; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_891) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_891; int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_860 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_860)); - *(reinterpret_cast(_gst_860) + (0)) = pq_13[0]; + auto _gst_892 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_892)); + *(reinterpret_cast(_gst_892) + (0)) = pq_13[0]; } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_861; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_861) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_861; + uint32_t _prmt_893; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_893) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_893; int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_862 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_862)); - *(reinterpret_cast(_gst_862) + (0)) = pq_14[0]; + auto _gst_894 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_894)); + *(reinterpret_cast(_gst_894) + (0)) = pq_14[0]; } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_863; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_863) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_863; + uint32_t _prmt_895; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_895) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_895; int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_864 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_864)); - *(reinterpret_cast(_gst_864) + (0)) = pq_15[0]; + auto _gst_896 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_896)); + *(reinterpret_cast(_gst_896) + (0)) = pq_15[0]; } } int t_a_24 = 8 * th; @@ -6684,47 +6266,47 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_865; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_865) : "f"(s_a_2), "f"(nm_t_5)); + float _sub_897; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_897) : "f"(s_a_2), "f"(nm_t_5)); float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_865), "f"(pa_2)); - float _sub_866; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_866) : "f"(pa_2), "f"(nm_t_5)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_897), "f"(pa_2)); + float _sub_898; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_898) : "f"(pa_2), "f"(nm_t_5)); float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_866), "f"(s_a_2)); - float _abs_868; - asm("abs.f32 %0, %1;" : "=f"(_abs_868) : "f"(s_a_2)); - float _abs_869; - asm("abs.f32 %0, %1;" : "=f"(_abs_869) : "f"(pa_2)); - uint32_t _setp_867_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_867_bits) : "f"(_abs_868), "f"(_abs_869)); - bool _setp_867 = _setp_867_bits != 0; - float nm_err_5 = ((_setp_867) ? nm_es_5 : nm_ex_5); - float _add_870; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_870) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_870; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_898), "f"(s_a_2)); + float _abs_900; + asm("abs.f32 %0, %1;" : "=f"(_abs_900) : "f"(s_a_2)); + float _abs_901; + asm("abs.f32 %0, %1;" : "=f"(_abs_901) : "f"(pa_2)); + uint32_t _setp_899_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_899_bits) : "f"(_abs_900), "f"(_abs_901)); + bool _setp_899 = _setp_899_bits != 0; + float nm_err_5 = ((_setp_899) ? nm_es_5 : nm_ex_5); + float _add_902; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_902) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_902; s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_871; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_871) : "f"(s_b_2), "f"(nm_t_6)); + float _sub_903; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_903) : "f"(s_b_2), "f"(nm_t_6)); float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_871), "f"(pb_2)); - float _sub_872; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_872) : "f"(pb_2), "f"(nm_t_6)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_903), "f"(pb_2)); + float _sub_904; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_904) : "f"(pb_2), "f"(nm_t_6)); float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_872), "f"(s_b_2)); - float _abs_874; - asm("abs.f32 %0, %1;" : "=f"(_abs_874) : "f"(s_b_2)); - float _abs_875; - asm("abs.f32 %0, %1;" : "=f"(_abs_875) : "f"(pb_2)); - uint32_t _setp_873_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_873_bits) : "f"(_abs_874), "f"(_abs_875)); - bool _setp_873 = _setp_873_bits != 0; - float nm_err_6 = ((_setp_873) ? nm_es_6 : nm_ex_6); - float _add_876; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_876) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_876; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_904), "f"(s_b_2)); + float _abs_906; + asm("abs.f32 %0, %1;" : "=f"(_abs_906) : "f"(s_b_2)); + float _abs_907; + asm("abs.f32 %0, %1;" : "=f"(_abs_907) : "f"(pb_2)); + uint32_t _setp_905_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_905_bits) : "f"(_abs_906), "f"(_abs_907)); + bool _setp_905 = _setp_905_bits != 0; + float nm_err_6 = ((_setp_905) ? nm_es_6 : nm_ex_6); + float _add_908; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_908) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_908; s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -6763,60 +6345,60 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_878; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_878) : "f"(ts_s), "f"(ts_bb)); - float _sub_877; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_877) : "f"(s_a_3), "f"(_sub_878)); - float _sub_879; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_879) : "f"(other[0]), "f"(ts_bb)); + float _sub_910; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_910) : "f"(ts_s), "f"(ts_bb)); + float _sub_909; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_909) : "f"(s_a_3), "f"(_sub_910)); + float _sub_911; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_911) : "f"(other[0]), "f"(ts_bb)); float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_877), "f"(_sub_879)); - float _add_880; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_880) : "f"(c_a_3), "f"(other[1])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_909), "f"(_sub_911)); + float _add_912; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_912) : "f"(c_a_3), "f"(other[1])); float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_880), "f"(ts_e)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_912), "f"(ts_e)); float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_882; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_882) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_881; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_881) : "f"(ts_s), "f"(_sub_882)); - float _sub_883; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_883) : "f"(pm_c), "f"(ts_bb_0)); + float _sub_914; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_914) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_913; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_913) : "f"(ts_s), "f"(_sub_914)); + float _sub_915; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_915) : "f"(pm_c), "f"(ts_bb_0)); float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_881), "f"(_sub_883)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_913), "f"(_sub_915)); s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_885; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_885) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_884; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_884) : "f"(s_b_3), "f"(_sub_885)); - float _sub_886; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_886) : "f"(other[2]), "f"(ts_bb_1)); + float _sub_917; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_917) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_916; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_916) : "f"(s_b_3), "f"(_sub_917)); + float _sub_918; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_918) : "f"(other[2]), "f"(ts_bb_1)); float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_884), "f"(_sub_886)); - float _add_887; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_887) : "f"(c_b_3), "f"(other[3])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_916), "f"(_sub_918)); + float _add_919; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_919) : "f"(c_b_3), "f"(other[3])); float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_887), "f"(ts_e_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_919), "f"(ts_e_1)); float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_889; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_889) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_888; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_888) : "f"(ts_s_1), "f"(_sub_889)); - float _sub_890; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_890) : "f"(pm_c_0), "f"(ts_bb_2)); + float _sub_921; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_921) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_920; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_920) : "f"(ts_s_1), "f"(_sub_921)); + float _sub_922; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_922) : "f"(pm_c_0), "f"(ts_bb_2)); float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_888), "f"(_sub_890)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_920), "f"(_sub_922)); s_b_3 = ts_s_2; c_b_3 = ts_e_2; gacc[0] = s_a_3; @@ -6834,10 +6416,10 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM } if ((lane_1 & 3) == 0) { int e_star = lane_1 >> 2 & 7; - pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_1706; - pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1484; + pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_2; + pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1; pool_f32[28160 + qd * 64 + 32 + 8 * th + e_star] = v1; - pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_1241; + pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_0; } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync 1, 256;" ::: "memory"); @@ -6927,41 +6509,41 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + for (int r_284 = 0; r_284 < 16; r_284++) { + int t_3 = r_284; + if (t_3 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -7033,41 +6615,41 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_24 = 16 + r_348; - if (t_24 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + for (int r_285 = 0; r_285 < 16; r_285++) { + int t_4 = 16 + r_285; + if (t_4 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -7183,41 +6765,41 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_25 = 32 + r_349; - if (t_25 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + for (int r_286 = 0; r_286 < 16; r_286++) { + int t_5 = 32 + r_286; + if (t_5 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -7365,41 +6947,41 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_26 = 48 + r_350; - if (t_26 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + for (int r_287 = 0; r_287 < 16; r_287++) { + int t_6 = 48 + r_287; + if (t_6 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_binding.cu similarity index 98% rename from flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu rename to flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_binding.cu index 9790bcabdfd..73779f19232 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_kda_det_train_328afe43951979e798d1, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_kda_det_train_a135a53a639f33f921ab, for the device source that defines it. Do not edit. #include #include #include @@ -23,13 +23,13 @@ #include -extern "C" __global__ void kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride); +extern "C" __global__ void kernel_cake_kda_det_train_a135a53a639f33f921ab(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride); -namespace cake_binding_kernel_cake_kda_det_train_328afe43951979e798d1 { +namespace cake_binding_kernel_cake_kda_det_train_a135a53a639f33f921ab { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_kda_det_train_328afe43951979e798d1"; +constexpr const char* KERNEL_NAME = "kernel_cake_kda_det_train_a135a53a639f33f921ab"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -487,7 +487,7 @@ void Run(TensorView a_dAqk, TensorView a_dAkk, TensorView a_gk, TensorView a_k_e const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { CheckCuda(cudaSetDevice(dev.device_id), "cudaSetDevice"); - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_kda_det_train_328afe43951979e798d1), cudaFuncAttributeMaxDynamicSharedMemorySize, 115200), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_kda_det_train_a135a53a639f33f921ab), cudaFuncAttributeMaxDynamicSharedMemorySize, 115200), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -527,11 +527,11 @@ void Run(TensorView a_dAqk, TensorView a_dAkk, TensorView a_gk, TensorView a_k_e config.blockDim = dim3(288, 1, 1); config.dynamicSmemBytes = 115200; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_kda_det_train_328afe43951979e798d1), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_kda_det_train_a135a53a639f33f921ab), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_kda_det_train_328afe43951979e798d1 +} // namespace cake_binding_kernel_cake_kda_det_train_a135a53a639f33f921ab -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_kda_det_train_328afe43951979e798d1::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_kda_det_train_a135a53a639f33f921ab::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_kernel.cu similarity index 90% rename from flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_kernel.cu rename to flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_kernel.cu index 05f7949c75c..5744991ab5f 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 add_f32x2_noftz(float2 a, float2 b) { float2 r; asm("add.rn.f32x2 %0, %1, %2;" @@ -207,10 +214,26 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void -kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride) +kernel_cake_kda_det_train_a135a53a639f33f921ab(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -2275,12 +2298,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_3 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * e2_3); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_3, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_3 : __fmaf_rn(dl, _rcp_170, e2_3)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2304,12 +2333,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_4 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * e2_4); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_4, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_4, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_4 : __fmaf_rn(dl_1, _rcp_174, e2_4)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2333,12 +2368,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_5 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * e2_5); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_5, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_5, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_5 : __fmaf_rn(dl_3, _rcp_178, e2_5)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2362,12 +2403,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_6 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * e2_6); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_6, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_6, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_6 : __fmaf_rn(dl_5, _rcp_182, e2_6)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2403,12 +2450,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_2 = 48 + 8 * th + 2; @@ -2428,12 +2481,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_1 = approx_exp2(dc2_2.x); float e2_1_1 = approx_exp2(dc2_2.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_190 = approx_rcp(k_t0_2); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_191 = approx_rcp(k_t1_2); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_2.x; pan_641[3] = p2_2.y; int t0_3 = 48 + 8 * th + 4; @@ -2453,12 +2512,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_2 = approx_exp2(dc2_3.x); float e2_1_2 = approx_exp2(dc2_3.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_194 = approx_rcp(k_t0_3); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_195 = approx_rcp(k_t1_3); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_3.x; pan_641[5] = p2_3.y; int t0_4 = 48 + 8 * th + 6; @@ -2478,12 +2543,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(_f2_61, make_float2(g_t0_4, g_t1_4)); float e2_0_3 = approx_exp2(dc2_4.x); float e2_1_3 = approx_exp2(dc2_4.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_198 = approx_rcp(k_t0_4); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_199 = approx_rcp(k_t1_4); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_4.x; pan_641[7] = p2_4.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2531,12 +2602,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_202 = approx_rcp(k_t0_5); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_203 = approx_rcp(k_t1_5); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_5.x; pan_703[1] = p2_5.y; int t0_6 = 8 * th + 2; @@ -2560,12 +2637,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_2.x); float en_1_1 = approx_exp2(dn2_2.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_206 = approx_rcp(k_t0_6); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_207 = approx_rcp(k_t1_6); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_6.x; pan_703[3] = p2_6.y; int t0_7 = 8 * th + 4; @@ -2589,12 +2672,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_3.x); float en_1_2 = approx_exp2(dn2_3.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_210 = approx_rcp(k_t0_7); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_211 = approx_rcp(k_t1_7); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_7.x; pan_703[5] = p2_7.y; int t0_8 = 8 * th + 6; @@ -2618,12 +2707,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_4.x); float en_1_3 = approx_exp2(dn2_4.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_214 = approx_rcp(k_t0_8); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_215 = approx_rcp(k_t1_8); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_8.x; pan_703[7] = p2_8.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2665,12 +2760,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_5.x); float en_1_4 = approx_exp2(dn2_5.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_218 = approx_rcp(k_t0_9); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_219 = approx_rcp(k_t1_9); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_9.x; pan_777[1] = p2_9.y; int t0_10 = 16 + 8 * th + 2; @@ -2694,12 +2795,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_6.x); float en_1_5 = approx_exp2(dn2_6.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_222 = approx_rcp(k_t0_10); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_223 = approx_rcp(k_t1_10); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_10.x; pan_777[3] = p2_10.y; int t0_11 = 16 + 8 * th + 4; @@ -2723,12 +2830,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_7.x); float en_1_6 = approx_exp2(dn2_7.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_226 = approx_rcp(k_t0_11); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_227 = approx_rcp(k_t1_11); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_11.x; pan_777[5] = p2_11.y; int t0_12 = 16 + 8 * th + 6; @@ -2752,12 +2865,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_8 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_8.x); float en_1_7 = approx_exp2(dn2_8.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_230 = approx_rcp(k_t0_12); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_231 = approx_rcp(k_t1_12); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_12.x; pan_777[7] = p2_12.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2828,37 +2947,37 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_4 = approx_exp2(d2_57.x); float e1_4 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_4, e1_4); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_4, e1_4); float2 v2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_4.x; v_562_0[1] = v2_4.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_5 = approx_exp2(d2_58.x); float e1_5 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_5, e1_5); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_5, e1_5); float2 v2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_5.x; v_562_0[3] = v2_5.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_6 = approx_exp2(d2_59.x); float e1_6 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_6, e1_6); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_6, e1_6); float2 v2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_6.x; v_562_0[5] = v2_6.y; float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_7 = approx_exp2(d2_60.x); float e1_7 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_7, e1_7); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_7, e1_7); float2 v2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_7.x; v_562_0[7] = v2_7.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2870,29 +2989,29 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2900,29 +3019,29 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2992,7 +3111,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3024,7 +3148,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3056,7 +3185,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_2)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3088,7 +3222,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_3)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3120,7 +3259,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_4)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3152,7 +3296,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_5)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3184,7 +3333,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_6)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3216,7 +3370,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_7)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3231,31 +3390,31 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_216 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_216; + float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_248; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_217 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_217; + float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_249; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_218 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_218; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_250; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_219 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_219; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_251; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_220 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_220; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_252; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_221 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_221; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_253; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_222 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_222; - float _shfl_xor_sync_223 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_223; - float _shfl_xor_sync_224 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_224; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_254; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_255; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_256; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); int ew_0 = 98304 - 32768 * th >> 2; @@ -3267,8 +3426,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int qq = lane_1 >> 1 & 3; int j = qd + 4 * tb; bool live = 32 + 8 * th + j < clen; - int _min_225 = ((32 + 8 * th + j) < (clen - 1) ? (32 + 8 * th + j) : (clen - 1)); - long long row = ext0 + (long long)_min_225; + int _min_257 = ((32 + 8 * th + j) < (clen - 1) ? (32 + 8 * th + j) : (clen - 1)); + long long row = ext0 + (long long)_min_257; bool is_k = tk == 1; __nv_bfloat16* xr = ((is_k) ? (k_raw) : (q_raw)); int xts = ((is_k) ? k_raw_row_stride : q_raw_row_stride); @@ -3295,43 +3454,43 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (sb + 8 * (3 ^ qq))))); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint2 _vld_226; - _vld_226 = *reinterpret_cast(xr + (xrow + (long long)(8 * (0 ^ qq))) + 0); - uint32_t* _vpairs_226 = reinterpret_cast(&_vld_226); + uint2 _vld_258; + _vld_258 = *reinterpret_cast(xr + (xrow + (long long)(8 * (0 ^ qq))) + 0); + uint32_t* _vpairs_258 = reinterpret_cast(&_vld_258); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_226[_pair]) << 16); - (&xv[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_226[_pair]) & 0xffff0000u); + (&xv[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_258[_pair]) << 16); + (&xv[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_258[_pair]) & 0xffff0000u); } } { - uint2 _vld_227; - _vld_227 = *reinterpret_cast(xr + (xrow + (long long)(8 * (1 ^ qq))) + 0); - uint32_t* _vpairs_227 = reinterpret_cast(&_vld_227); + uint2 _vld_259; + _vld_259 = *reinterpret_cast(xr + (xrow + (long long)(8 * (1 ^ qq))) + 0); + uint32_t* _vpairs_259 = reinterpret_cast(&_vld_259); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_227[_pair]) << 16); - (&xv[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_227[_pair]) & 0xffff0000u); + (&xv[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_259[_pair]) << 16); + (&xv[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_259[_pair]) & 0xffff0000u); } } { - uint2 _vld_228; - _vld_228 = *reinterpret_cast(xr + (xrow + (long long)(8 * (2 ^ qq))) + 0); - uint32_t* _vpairs_228 = reinterpret_cast(&_vld_228); + uint2 _vld_260; + _vld_260 = *reinterpret_cast(xr + (xrow + (long long)(8 * (2 ^ qq))) + 0); + uint32_t* _vpairs_260 = reinterpret_cast(&_vld_260); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_228[_pair]) << 16); - (&xv[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_228[_pair]) & 0xffff0000u); + (&xv[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_260[_pair]) << 16); + (&xv[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_260[_pair]) & 0xffff0000u); } } { - uint2 _vld_229; - _vld_229 = *reinterpret_cast(xr + (xrow + (long long)(8 * (3 ^ qq))) + 0); - uint32_t* _vpairs_229 = reinterpret_cast(&_vld_229); + uint2 _vld_261; + _vld_261 = *reinterpret_cast(xr + (xrow + (long long)(8 * (3 ^ qq))) + 0); + uint32_t* _vpairs_261 = reinterpret_cast(&_vld_261); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_229[_pair]) << 16); - (&xv[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_229[_pair]) & 0xffff0000u); + (&xv[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_261[_pair]) << 16); + (&xv[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_261[_pair]) & 0xffff0000u); } } { @@ -3340,65 +3499,65 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 r2 = make_float2(rv[0], rv[0]); float y[16]; float p[16]; - float2 _f32x2_mul_a_230 = make_float2(xv[0], xv[1]); + float2 _f32x2_mul_a_262 = make_float2(xv[0], xv[1]); float2 y2_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_230), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_262), "l"(*(const unsigned long long*)&r2)); y[0] = y2_0.x; y[1] = y2_0.y; float2 p2_13 = fma_f32x2_rn_noftz(make_float2(g[0], g[1]), y2_0, nz2); p[0] = p2_13.x; p[1] = p2_13.y; - float2 _f32x2_mul_a_231 = make_float2(xv[2], xv[3]); + float2 _f32x2_mul_a_263 = make_float2(xv[2], xv[3]); float2 y2_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_231), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_263), "l"(*(const unsigned long long*)&r2)); y[2] = y2_1.x; y[3] = y2_1.y; float2 p2_14 = fma_f32x2_rn_noftz(make_float2(g[2], g[3]), y2_1, nz2); p[2] = p2_14.x; p[3] = p2_14.y; - float2 _f32x2_mul_a_232 = make_float2(xv[4], xv[5]); + float2 _f32x2_mul_a_264 = make_float2(xv[4], xv[5]); float2 y2_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_264), "l"(*(const unsigned long long*)&r2)); y[4] = y2_2.x; y[5] = y2_2.y; float2 p2_15 = fma_f32x2_rn_noftz(make_float2(g[4], g[5]), y2_2, nz2); p[4] = p2_15.x; p[5] = p2_15.y; - float2 _f32x2_mul_a_233 = make_float2(xv[6], xv[7]); + float2 _f32x2_mul_a_265 = make_float2(xv[6], xv[7]); float2 y2_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_233), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_265), "l"(*(const unsigned long long*)&r2)); y[6] = y2_3.x; y[7] = y2_3.y; float2 p2_16 = fma_f32x2_rn_noftz(make_float2(g[6], g[7]), y2_3, nz2); p[6] = p2_16.x; p[7] = p2_16.y; - float2 _f32x2_mul_a_234 = make_float2(xv[8], xv[9]); + float2 _f32x2_mul_a_266 = make_float2(xv[8], xv[9]); float2 y2_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_266), "l"(*(const unsigned long long*)&r2)); y[8] = y2_4.x; y[9] = y2_4.y; float2 p2_17 = fma_f32x2_rn_noftz(make_float2(g[8], g[9]), y2_4, nz2); p[8] = p2_17.x; p[9] = p2_17.y; - float2 _f32x2_mul_a_235 = make_float2(xv[10], xv[11]); + float2 _f32x2_mul_a_267 = make_float2(xv[10], xv[11]); float2 y2_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_235), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&r2)); y[10] = y2_5.x; y[11] = y2_5.y; float2 p2_18 = fma_f32x2_rn_noftz(make_float2(g[10], g[11]), y2_5, nz2); p[10] = p2_18.x; p[11] = p2_18.y; - float2 _f32x2_mul_a_236 = make_float2(xv[12], xv[13]); + float2 _f32x2_mul_a_268 = make_float2(xv[12], xv[13]); float2 y2_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_268), "l"(*(const unsigned long long*)&r2)); y[12] = y2_6.x; y[13] = y2_6.y; float2 p2_19 = fma_f32x2_rn_noftz(make_float2(g[12], g[13]), y2_6, nz2); p[12] = p2_19.x; p[13] = p2_19.y; - float2 _f32x2_mul_a_237 = make_float2(xv[14], xv[15]); + float2 _f32x2_mul_a_269 = make_float2(xv[14], xv[15]); float2 y2_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_237), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_269), "l"(*(const unsigned long long*)&r2)); y[14] = y2_7.x; y[15] = y2_7.y; float2 p2_20 = fma_f32x2_rn_noftz(make_float2(g[14], g[15]), y2_7, nz2); @@ -3415,35 +3574,35 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 e2_8 = add_f32x2_noftz(a2_1, c2_1); e[2] = e2_8.x; e[3] = e2_8.y; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, e[0], 1); - float _add_238; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_238) : "f"(e[0]), "f"(_shfl_xor_sync_239)); - e[0] = _add_238; - float _shfl_xor_sync_241 = __shfl_xor_sync(0xFFFFFFFF, e[1], 1); - float _add_240; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_240) : "f"(e[1]), "f"(_shfl_xor_sync_241)); - e[1] = _add_240; - float _shfl_xor_sync_243 = __shfl_xor_sync(0xFFFFFFFF, e[2], 1); - float _add_242; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_242) : "f"(e[2]), "f"(_shfl_xor_sync_243)); - e[2] = _add_242; - float _shfl_xor_sync_245 = __shfl_xor_sync(0xFFFFFFFF, e[3], 1); - float _add_244; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_244) : "f"(e[3]), "f"(_shfl_xor_sync_245)); - e[3] = _add_244; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, e[0], 1); + float _add_270; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_270) : "f"(e[0]), "f"(_shfl_xor_sync_271)); + e[0] = _add_270; + float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, e[1], 1); + float _add_272; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_272) : "f"(e[1]), "f"(_shfl_xor_sync_273)); + e[1] = _add_272; + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, e[2], 1); + float _add_274; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_274) : "f"(e[2]), "f"(_shfl_xor_sync_275)); + e[2] = _add_274; + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, e[3], 1); + float _add_276; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_276) : "f"(e[3]), "f"(_shfl_xor_sync_277)); + e[3] = _add_276; float2 f2 = add_f32x2_noftz(make_float2(e[0], e[1]), make_float2(e[2], e[3])); float sq; asm("add.rn.f32 %0, %1, %2;" : "=f"(sq) : "f"(f2.x), "f"(f2.y)); - float _shfl_sync_248 = __shfl_sync(0xFFFFFFFF, sq, 0, 8); - float _shfl_sync_249 = __shfl_sync(0xFFFFFFFF, sq, 2, 8); - float _add_247; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_247) : "f"(_shfl_sync_248), "f"(_shfl_sync_249)); - float _shfl_sync_250 = __shfl_sync(0xFFFFFFFF, sq, 4, 8); - float _add_246; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_246) : "f"(_add_247), "f"(_shfl_sync_250)); - float _shfl_sync_251 = __shfl_sync(0xFFFFFFFF, sq, 6, 8); + float _shfl_sync_280 = __shfl_sync(0xFFFFFFFF, sq, 0, 8); + float _shfl_sync_281 = __shfl_sync(0xFFFFFFFF, sq, 2, 8); + float _add_279; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_279) : "f"(_shfl_sync_280), "f"(_shfl_sync_281)); + float _shfl_sync_282 = __shfl_sync(0xFFFFFFFF, sq, 4, 8); + float _add_278; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_278) : "f"(_add_279), "f"(_shfl_sync_282)); + float _shfl_sync_283 = __shfl_sync(0xFFFFFFFF, sq, 6, 8); float dot; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_246), "f"(_shfl_sync_251)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_278), "f"(_shfl_sync_283)); float2 d2 = make_float2(dot, dot); float o_0[4]; float2 t2_0 = fma_f32x2_rn_noftz(make_float2(y[0], y[1]), d2, nz2); @@ -3581,7 +3740,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; + float qe_7 = q_t_1_8 * e_dg_0; + float kp_19 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_0; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_14[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_0) * _tmem_load_15[7]); + float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_16[7], k_t_0, cr_8)) + dgf_0; s_run_1059[0] = s_run_1059[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3613,7 +3777,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[6]; - float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, (dk2b_2 - _tmem_load_16[6]) * k_t_2) + dgf_2; + float qe_8 = q_t_1_9 * e_dg_2; + float kp_20 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_14[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_2) * _tmem_load_15[6]); + float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_16[6], k_t_2, cr_9)) + dgf_2; s_run_1059[0] = s_run_1059[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3645,7 +3814,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[5]; - float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, (dk2b_3 - _tmem_load_16[5]) * k_t_3) + dgf_3; + float qe_9 = q_t_1_10 * e_dg_3; + float kp_21 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_3; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_14[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_3) * _tmem_load_15[5]); + float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_16[5], k_t_3, cr_10)) + dgf_3; s_run_1059[0] = s_run_1059[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3677,7 +3851,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[4]; - float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, (dk2b_4 - _tmem_load_16[4]) * k_t_4) + dgf_4; + float qe_10 = q_t_1_11 * e_dg_4; + float kp_22 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_4; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_14[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_4) * _tmem_load_15[4]); + float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_16[4], k_t_4, cr_11)) + dgf_4; s_run_1059[0] = s_run_1059[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3709,7 +3888,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[3]; - float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, (dk2b_5 - _tmem_load_16[3]) * k_t_5) + dgf_5; + float qe_11 = q_t_1_12 * e_dg_5; + float kp_23 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_5; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_14[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_5) * _tmem_load_15[3]); + float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_16[3], k_t_5, cr_12)) + dgf_5; s_run_1059[0] = s_run_1059[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3741,7 +3925,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[2]; - float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, (dk2b_6 - _tmem_load_16[2]) * k_t_6) + dgf_6; + float qe_12 = q_t_1_13 * e_dg_6; + float kp_24 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_6; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_14[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_6) * _tmem_load_15[2]); + float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_16[2], k_t_6, cr_13)) + dgf_6; s_run_1059[0] = s_run_1059[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3773,7 +3962,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[1]; - float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, (dk2b_7 - _tmem_load_16[1]) * k_t_7) + dgf_7; + float qe_13 = q_t_1_14 * e_dg_7; + float kp_25 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_7; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_14[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_7) * _tmem_load_15[1]); + float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_16[1], k_t_7, cr_14)) + dgf_7; s_run_1059[0] = s_run_1059[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3805,7 +3999,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_16[0]; - float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, (dk2b_8 - _tmem_load_16[0]) * k_t_8) + dgf_8; + float qe_14 = q_t_1_15 * e_dg_8; + float kp_26 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_8; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_14[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_8) * _tmem_load_15[0]); + float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_16[0], k_t_8, cr_15)) + dgf_8; s_run_1059[0] = s_run_1059[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3820,31 +4019,31 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int b2_1231 = lane_1 >> 2 & 1; float v4_1232[4]; float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_252; + float _shfl_xor_sync_284 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_284; float send_2 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_253; + float _shfl_xor_sync_285 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_285; float send_3 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_254; + float _shfl_xor_sync_286 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_286; float send_4 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_255; + float _shfl_xor_sync_287 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_287; float v2_1237[2]; float send_5 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_256; + float _shfl_xor_sync_288 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_288; float send_6 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); + v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_289; float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_258; - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_259; - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_260; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); + float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_290; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); + v1_1241 = v1_1241 + _shfl_xor_sync_291; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); + v1_1241 = v1_1241 + _shfl_xor_sync_292; asm volatile("tcgen05.fence::before_thread_sync;"); int ew_1 = 98304 - 32768 * th >> 2; float nz_0 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); @@ -3855,8 +4054,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int qq_0 = lane_1 >> 1 & 3; int j_0 = qd + 4 * tb_0; bool live_0 = 48 + 8 * th + j_0 < clen; - int _min_261 = ((48 + 8 * th + j_0) < (clen - 1) ? (48 + 8 * th + j_0) : (clen - 1)); - long long row_0 = ext0 + (long long)_min_261; + int _min_293 = ((48 + 8 * th + j_0) < (clen - 1) ? (48 + 8 * th + j_0) : (clen - 1)); + long long row_0 = ext0 + (long long)_min_293; bool is_k_0 = tk_0 == 1; __nv_bfloat16* xr_0 = ((is_k_0) ? (k_raw) : (q_raw)); int xts_0 = ((is_k_0) ? k_raw_row_stride : q_raw_row_stride); @@ -3883,43 +4082,43 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (sb_0 + 8 * (3 ^ qq_0))))); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint2 _vld_262; - _vld_262 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (0 ^ qq_0))) + 0); - uint32_t* _vpairs_262 = reinterpret_cast(&_vld_262); + uint2 _vld_294; + _vld_294 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (0 ^ qq_0))) + 0); + uint32_t* _vpairs_294 = reinterpret_cast(&_vld_294); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_262[_pair]) << 16); - (&xv_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_262[_pair]) & 0xffff0000u); + (&xv_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_294[_pair]) << 16); + (&xv_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_294[_pair]) & 0xffff0000u); } } { - uint2 _vld_263; - _vld_263 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (1 ^ qq_0))) + 0); - uint32_t* _vpairs_263 = reinterpret_cast(&_vld_263); + uint2 _vld_295; + _vld_295 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (1 ^ qq_0))) + 0); + uint32_t* _vpairs_295 = reinterpret_cast(&_vld_295); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_0[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_263[_pair]) << 16); - (&xv_0[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_263[_pair]) & 0xffff0000u); + (&xv_0[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_295[_pair]) << 16); + (&xv_0[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_295[_pair]) & 0xffff0000u); } } { - uint2 _vld_264; - _vld_264 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (2 ^ qq_0))) + 0); - uint32_t* _vpairs_264 = reinterpret_cast(&_vld_264); + uint2 _vld_296; + _vld_296 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (2 ^ qq_0))) + 0); + uint32_t* _vpairs_296 = reinterpret_cast(&_vld_296); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_0[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_264[_pair]) << 16); - (&xv_0[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_264[_pair]) & 0xffff0000u); + (&xv_0[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_296[_pair]) << 16); + (&xv_0[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_296[_pair]) & 0xffff0000u); } } { - uint2 _vld_265; - _vld_265 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (3 ^ qq_0))) + 0); - uint32_t* _vpairs_265 = reinterpret_cast(&_vld_265); + uint2 _vld_297; + _vld_297 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (3 ^ qq_0))) + 0); + uint32_t* _vpairs_297 = reinterpret_cast(&_vld_297); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_0[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_265[_pair]) << 16); - (&xv_0[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_265[_pair]) & 0xffff0000u); + (&xv_0[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_297[_pair]) << 16); + (&xv_0[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_297[_pair]) & 0xffff0000u); } } { @@ -3928,65 +4127,65 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 r2_0 = make_float2(rv_0[0], rv_0[0]); float y_0[16]; float p_0[16]; - float2 _f32x2_mul_a_266 = make_float2(xv_0[0], xv_0[1]); + float2 _f32x2_mul_a_298 = make_float2(xv_0[0], xv_0[1]); float2 y2_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_266), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_298), "l"(*(const unsigned long long*)&r2_0)); y_0[0] = y2_8.x; y_0[1] = y2_8.y; float2 p2_21 = fma_f32x2_rn_noftz(make_float2(g_0[0], g_0[1]), y2_8, nz2_0); p_0[0] = p2_21.x; p_0[1] = p2_21.y; - float2 _f32x2_mul_a_267 = make_float2(xv_0[2], xv_0[3]); + float2 _f32x2_mul_a_299 = make_float2(xv_0[2], xv_0[3]); float2 y2_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_299), "l"(*(const unsigned long long*)&r2_0)); y_0[2] = y2_9.x; y_0[3] = y2_9.y; float2 p2_22 = fma_f32x2_rn_noftz(make_float2(g_0[2], g_0[3]), y2_9, nz2_0); p_0[2] = p2_22.x; p_0[3] = p2_22.y; - float2 _f32x2_mul_a_268 = make_float2(xv_0[4], xv_0[5]); + float2 _f32x2_mul_a_300 = make_float2(xv_0[4], xv_0[5]); float2 y2_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_268), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&r2_0)); y_0[4] = y2_10.x; y_0[5] = y2_10.y; float2 p2_23 = fma_f32x2_rn_noftz(make_float2(g_0[4], g_0[5]), y2_10, nz2_0); p_0[4] = p2_23.x; p_0[5] = p2_23.y; - float2 _f32x2_mul_a_269 = make_float2(xv_0[6], xv_0[7]); + float2 _f32x2_mul_a_301 = make_float2(xv_0[6], xv_0[7]); float2 y2_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_269), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_301), "l"(*(const unsigned long long*)&r2_0)); y_0[6] = y2_11.x; y_0[7] = y2_11.y; float2 p2_24 = fma_f32x2_rn_noftz(make_float2(g_0[6], g_0[7]), y2_11, nz2_0); p_0[6] = p2_24.x; p_0[7] = p2_24.y; - float2 _f32x2_mul_a_270 = make_float2(xv_0[8], xv_0[9]); + float2 _f32x2_mul_a_302 = make_float2(xv_0[8], xv_0[9]); float2 y2_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_270), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_302), "l"(*(const unsigned long long*)&r2_0)); y_0[8] = y2_12.x; y_0[9] = y2_12.y; float2 p2_25 = fma_f32x2_rn_noftz(make_float2(g_0[8], g_0[9]), y2_12, nz2_0); p_0[8] = p2_25.x; p_0[9] = p2_25.y; - float2 _f32x2_mul_a_271 = make_float2(xv_0[10], xv_0[11]); + float2 _f32x2_mul_a_303 = make_float2(xv_0[10], xv_0[11]); float2 y2_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_271), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_303), "l"(*(const unsigned long long*)&r2_0)); y_0[10] = y2_13.x; y_0[11] = y2_13.y; float2 p2_26 = fma_f32x2_rn_noftz(make_float2(g_0[10], g_0[11]), y2_13, nz2_0); p_0[10] = p2_26.x; p_0[11] = p2_26.y; - float2 _f32x2_mul_a_272 = make_float2(xv_0[12], xv_0[13]); + float2 _f32x2_mul_a_304 = make_float2(xv_0[12], xv_0[13]); float2 y2_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_272), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_304), "l"(*(const unsigned long long*)&r2_0)); y_0[12] = y2_14.x; y_0[13] = y2_14.y; float2 p2_27 = fma_f32x2_rn_noftz(make_float2(g_0[12], g_0[13]), y2_14, nz2_0); p_0[12] = p2_27.x; p_0[13] = p2_27.y; - float2 _f32x2_mul_a_273 = make_float2(xv_0[14], xv_0[15]); + float2 _f32x2_mul_a_305 = make_float2(xv_0[14], xv_0[15]); float2 y2_15; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_273), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_305), "l"(*(const unsigned long long*)&r2_0)); y_0[14] = y2_15.x; y_0[15] = y2_15.y; float2 p2_28 = fma_f32x2_rn_noftz(make_float2(g_0[14], g_0[15]), y2_15, nz2_0); @@ -4003,35 +4202,35 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 e2_10 = add_f32x2_noftz(a2_3, c2_3); e_0[2] = e2_10.x; e_0[3] = e2_10.y; - float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, e_0[0], 1); - float _add_274; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_274) : "f"(e_0[0]), "f"(_shfl_xor_sync_275)); - e_0[0] = _add_274; - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, e_0[1], 1); - float _add_276; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_276) : "f"(e_0[1]), "f"(_shfl_xor_sync_277)); - e_0[1] = _add_276; - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, e_0[2], 1); - float _add_278; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_278) : "f"(e_0[2]), "f"(_shfl_xor_sync_279)); - e_0[2] = _add_278; - float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, e_0[3], 1); - float _add_280; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_280) : "f"(e_0[3]), "f"(_shfl_xor_sync_281)); - e_0[3] = _add_280; + float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, e_0[0], 1); + float _add_306; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_306) : "f"(e_0[0]), "f"(_shfl_xor_sync_307)); + e_0[0] = _add_306; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, e_0[1], 1); + float _add_308; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_308) : "f"(e_0[1]), "f"(_shfl_xor_sync_309)); + e_0[1] = _add_308; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, e_0[2], 1); + float _add_310; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_310) : "f"(e_0[2]), "f"(_shfl_xor_sync_311)); + e_0[2] = _add_310; + float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, e_0[3], 1); + float _add_312; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_312) : "f"(e_0[3]), "f"(_shfl_xor_sync_313)); + e_0[3] = _add_312; float2 f2_0 = add_f32x2_noftz(make_float2(e_0[0], e_0[1]), make_float2(e_0[2], e_0[3])); float sq_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_0) : "f"(f2_0.x), "f"(f2_0.y)); - float _shfl_sync_284 = __shfl_sync(0xFFFFFFFF, sq_0, 0, 8); - float _shfl_sync_285 = __shfl_sync(0xFFFFFFFF, sq_0, 2, 8); - float _add_283; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_283) : "f"(_shfl_sync_284), "f"(_shfl_sync_285)); - float _shfl_sync_286 = __shfl_sync(0xFFFFFFFF, sq_0, 4, 8); - float _add_282; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_282) : "f"(_add_283), "f"(_shfl_sync_286)); - float _shfl_sync_287 = __shfl_sync(0xFFFFFFFF, sq_0, 6, 8); + float _shfl_sync_316 = __shfl_sync(0xFFFFFFFF, sq_0, 0, 8); + float _shfl_sync_317 = __shfl_sync(0xFFFFFFFF, sq_0, 2, 8); + float _add_315; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_315) : "f"(_shfl_sync_316), "f"(_shfl_sync_317)); + float _shfl_sync_318 = __shfl_sync(0xFFFFFFFF, sq_0, 4, 8); + float _add_314; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_314) : "f"(_add_315), "f"(_shfl_sync_318)); + float _shfl_sync_319 = __shfl_sync(0xFFFFFFFF, sq_0, 6, 8); float dot_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_282), "f"(_shfl_sync_287)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_314), "f"(_shfl_sync_319)); float2 d2_61 = make_float2(dot_0, dot_0); float o_4[4]; float2 t2_8 = fma_f32x2_rn_noftz(make_float2(y_0[0], y_0[1]), d2_61, nz2_0); @@ -4114,8 +4313,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM } asm volatile("{ .reg .pred p_; setp.ne.b32 p_, %3, 0; @p_ st.global.v2.b32 [%0], {%1, %2}; }" :: "l"(dxp_0 + (orow_0 + (long long)(8 * (3 ^ qq_0)))), "r"(ob_7[(0) + 0]), "r"(ob_7[(0) + 1]), "r"((unsigned int)(live_0)) : "memory"); { - uint32_t _addr_288 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_288), "f"(tot_1228) : "memory"); + uint32_t _addr_320 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_320), "f"(tot_1228) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4144,9 +4343,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_0 = offset + _tmem_load_17[0]; if (t_9 < clen) { { - auto _gst_289 = dg_base + goff_1_0; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast(_gst_289) + (0)) = dg_1_0; + auto _gst_321 = dg_base + goff_1_0; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast(_gst_321) + (0)) = dg_1_0; } } int t_10 = 48 + 8 * th + 1; @@ -4154,9 +4353,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_1 = offset + _tmem_load_17[1]; if (t_10 < clen) { { - auto _gst_290 = dg_base + goff_1_1; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast(_gst_290) + (0)) = dg_1_1; + auto _gst_322 = dg_base + goff_1_1; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast(_gst_322) + (0)) = dg_1_1; } } int t_11 = 48 + 8 * th + 2; @@ -4164,9 +4363,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_2 = offset + _tmem_load_17[2]; if (t_11 < clen) { { - auto _gst_291 = dg_base + goff_1_2; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast(_gst_291) + (0)) = dg_1_2; + auto _gst_323 = dg_base + goff_1_2; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast(_gst_323) + (0)) = dg_1_2; } } int t_12 = 48 + 8 * th + 3; @@ -4174,9 +4373,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_3 = offset + _tmem_load_17[3]; if (t_12 < clen) { { - auto _gst_292 = dg_base + goff_1_3; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast(_gst_292) + (0)) = dg_1_3; + auto _gst_324 = dg_base + goff_1_3; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast(_gst_324) + (0)) = dg_1_3; } } int t_13 = 48 + 8 * th + 4; @@ -4184,9 +4383,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_4 = offset + _tmem_load_17[4]; if (t_13 < clen) { { - auto _gst_293 = dg_base + goff_1_4; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast(_gst_293) + (0)) = dg_1_4; + auto _gst_325 = dg_base + goff_1_4; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast(_gst_325) + (0)) = dg_1_4; } } int t_14 = 48 + 8 * th + 5; @@ -4194,9 +4393,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_5 = offset + _tmem_load_17[5]; if (t_14 < clen) { { - auto _gst_294 = dg_base + goff_1_5; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast(_gst_294) + (0)) = dg_1_5; + auto _gst_326 = dg_base + goff_1_5; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast(_gst_326) + (0)) = dg_1_5; } } int t_15 = 48 + 8 * th + 6; @@ -4204,9 +4403,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_6 = offset + _tmem_load_17[6]; if (t_15 < clen) { { - auto _gst_295 = dg_base + goff_1_6; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast(_gst_295) + (0)) = dg_1_6; + auto _gst_327 = dg_base + goff_1_6; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast(_gst_327) + (0)) = dg_1_6; } } int t_16 = 48 + 8 * th + 7; @@ -4214,9 +4413,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_7 = offset + _tmem_load_17[7]; if (t_16 < clen) { { - auto _gst_296 = dg_base + goff_1_7; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast(_gst_296) + (0)) = dg_1_7; + auto _gst_328 = dg_base + goff_1_7; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast(_gst_328) + (0)) = dg_1_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -4234,9 +4433,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_0 = offset_1250 + _tmem_load_19[0]; if (t_17 < clen) { { - auto _gst_297 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast(_gst_297) + (0)) = dg_0; + auto _gst_329 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast(_gst_329) + (0)) = dg_0; } } int t_18 = 32 + 8 * th + 1; @@ -4244,9 +4443,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1 = offset_1250 + _tmem_load_19[1]; if (t_18 < clen) { { - auto _gst_298 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast(_gst_298) + (0)) = dg_1; + auto _gst_330 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast(_gst_330) + (0)) = dg_1; } } int t_19 = 32 + 8 * th + 2; @@ -4254,9 +4453,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_2 = offset_1250 + _tmem_load_19[2]; if (t_19 < clen) { { - auto _gst_299 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast(_gst_299) + (0)) = dg_2; + auto _gst_331 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast(_gst_331) + (0)) = dg_2; } } int t_20 = 32 + 8 * th + 3; @@ -4264,9 +4463,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_3 = offset_1250 + _tmem_load_19[3]; if (t_20 < clen) { { - auto _gst_300 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_300)); - *(reinterpret_cast(_gst_300) + (0)) = dg_3; + auto _gst_332 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_332)); + *(reinterpret_cast(_gst_332) + (0)) = dg_3; } } int t_21 = 32 + 8 * th + 4; @@ -4274,9 +4473,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_4 = offset_1250 + _tmem_load_19[4]; if (t_21 < clen) { { - auto _gst_301 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_301)); - *(reinterpret_cast(_gst_301) + (0)) = dg_4; + auto _gst_333 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_333)); + *(reinterpret_cast(_gst_333) + (0)) = dg_4; } } int t_22 = 32 + 8 * th + 5; @@ -4284,9 +4483,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_5 = offset_1250 + _tmem_load_19[5]; if (t_22 < clen) { { - auto _gst_302 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_302)); - *(reinterpret_cast(_gst_302) + (0)) = dg_5; + auto _gst_334 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_334)); + *(reinterpret_cast(_gst_334) + (0)) = dg_5; } } int t_23 = 32 + 8 * th + 6; @@ -4294,9 +4493,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_6 = offset_1250 + _tmem_load_19[6]; if (t_23 < clen) { { - auto _gst_303 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_303)); - *(reinterpret_cast(_gst_303) + (0)) = dg_6; + auto _gst_335 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_335)); + *(reinterpret_cast(_gst_335) + (0)) = dg_6; } } int t_24 = 32 + 8 * th + 7; @@ -4304,15 +4503,15 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_7 = offset_1250 + _tmem_load_19[7]; if (t_24 < clen) { { - auto _gst_304 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_304)); - *(reinterpret_cast(_gst_304) + (0)) = dg_7; + auto _gst_336 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_336)); + *(reinterpret_cast(_gst_336) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi_1248 + t_lo_1249; { - uint32_t _addr_305 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_305), "f"(off_acc[0]) : "memory"); + uint32_t _addr_337 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_337), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4372,7 +4571,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[7]; - float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, (dk2b_9 - _tmem_load_24[7]) * k_t_9) + dgf_9; + float qe_15 = q_t_1_16 * e_dg_9; + float kp_27 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_9; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_9) * _tmem_load_23[7]); + float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_24[7], k_t_9, cr_16)) + dgf_9; s_run_1302[0] = s_run_1302[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4404,7 +4608,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[6]; - float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, (dk2b_10 - _tmem_load_24[6]) * k_t_10) + dgf_10; + float qe_16 = q_t_1_17 * e_dg_10; + float kp_28 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_10; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_10) * _tmem_load_23[6]); + float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_24[6], k_t_10, cr_17)) + dgf_10; s_run_1302[0] = s_run_1302[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4436,7 +4645,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[5]; - float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, (dk2b_11 - _tmem_load_24[5]) * k_t_11) + dgf_11; + float qe_17 = q_t_1_18 * e_dg_11; + float kp_29 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_11; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_11) * _tmem_load_23[5]); + float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_24[5], k_t_11, cr_18)) + dgf_11; s_run_1302[0] = s_run_1302[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4468,7 +4682,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[4]; - float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, (dk2b_12 - _tmem_load_24[4]) * k_t_12) + dgf_12; + float qe_18 = q_t_1_19 * e_dg_12; + float kp_30 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_12; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_12) * _tmem_load_23[4]); + float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_24[4], k_t_12, cr_19)) + dgf_12; s_run_1302[0] = s_run_1302[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4500,7 +4719,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[3]; - float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, (dk2b_13 - _tmem_load_24[3]) * k_t_13) + dgf_13; + float qe_19 = q_t_1_20 * e_dg_13; + float kp_31 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_13; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_13) * _tmem_load_23[3]); + float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_24[3], k_t_13, cr_20)) + dgf_13; s_run_1302[0] = s_run_1302[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4532,7 +4756,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[2]; - float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, (dk2b_14 - _tmem_load_24[2]) * k_t_14) + dgf_14; + float qe_20 = q_t_1_21 * e_dg_14; + float kp_32 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_14; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_14) * _tmem_load_23[2]); + float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_24[2], k_t_14, cr_21)) + dgf_14; s_run_1302[0] = s_run_1302[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4564,7 +4793,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[1]; - float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, (dk2b_15 - _tmem_load_24[1]) * k_t_15) + dgf_15; + float qe_21 = q_t_1_22 * e_dg_15; + float kp_33 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_15; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_15) * _tmem_load_23[1]); + float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_24[1], k_t_15, cr_22)) + dgf_15; s_run_1302[0] = s_run_1302[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4596,7 +4830,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_24[0]; - float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, (dk2b_16 - _tmem_load_24[0]) * k_t_16) + dgf_16; + float qe_22 = q_t_1_23 * e_dg_16; + float kp_34 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_16; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_16) * _tmem_load_23[0]); + float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, __fmaf_rn(dk2b_16 - _tmem_load_24[0], k_t_16, cr_23)) + dgf_16; s_run_1302[0] = s_run_1302[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4611,31 +4850,31 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; float send_7 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_306; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_338; float send_8 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_307; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_339; float send_9 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_308; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_340; float send_10 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_309; + float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_341; float v2_1480[2]; float send_11 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_310; + float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_342; float send_12 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_311; + float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_343; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_312; - float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_313; - float _shfl_xor_sync_314 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_314; + float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_344; + float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_345; + float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_346; asm volatile("tcgen05.fence::before_thread_sync;"); int ew_2 = 81920 - 32768 * th >> 2; float nz_1 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); @@ -4646,8 +4885,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int qq_1 = lane_1 >> 1 & 3; int j_1 = qd + 4 * tb_1; bool live_1 = 16 + 8 * th + j_1 < clen; - int _min_315 = ((16 + 8 * th + j_1) < (clen - 1) ? (16 + 8 * th + j_1) : (clen - 1)); - long long row_1 = ext0 + (long long)_min_315; + int _min_347 = ((16 + 8 * th + j_1) < (clen - 1) ? (16 + 8 * th + j_1) : (clen - 1)); + long long row_1 = ext0 + (long long)_min_347; bool is_k_1 = tk_1 == 1; __nv_bfloat16* xr_1 = ((is_k_1) ? (k_raw) : (q_raw)); int xts_1 = ((is_k_1) ? k_raw_row_stride : q_raw_row_stride); @@ -4674,43 +4913,43 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (sb_1 + 8 * (3 ^ qq_1))))); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); { - uint2 _vld_316; - _vld_316 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (0 ^ qq_1))) + 0); - uint32_t* _vpairs_316 = reinterpret_cast(&_vld_316); + uint2 _vld_348; + _vld_348 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (0 ^ qq_1))) + 0); + uint32_t* _vpairs_348 = reinterpret_cast(&_vld_348); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_316[_pair]) << 16); - (&xv_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_316[_pair]) & 0xffff0000u); + (&xv_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_348[_pair]) << 16); + (&xv_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_348[_pair]) & 0xffff0000u); } } { - uint2 _vld_317; - _vld_317 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (1 ^ qq_1))) + 0); - uint32_t* _vpairs_317 = reinterpret_cast(&_vld_317); + uint2 _vld_349; + _vld_349 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (1 ^ qq_1))) + 0); + uint32_t* _vpairs_349 = reinterpret_cast(&_vld_349); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_1[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_317[_pair]) << 16); - (&xv_1[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_317[_pair]) & 0xffff0000u); + (&xv_1[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_349[_pair]) << 16); + (&xv_1[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_349[_pair]) & 0xffff0000u); } } { - uint2 _vld_318; - _vld_318 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (2 ^ qq_1))) + 0); - uint32_t* _vpairs_318 = reinterpret_cast(&_vld_318); + uint2 _vld_350; + _vld_350 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (2 ^ qq_1))) + 0); + uint32_t* _vpairs_350 = reinterpret_cast(&_vld_350); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_1[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_318[_pair]) << 16); - (&xv_1[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_318[_pair]) & 0xffff0000u); + (&xv_1[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_350[_pair]) << 16); + (&xv_1[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_350[_pair]) & 0xffff0000u); } } { - uint2 _vld_319; - _vld_319 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (3 ^ qq_1))) + 0); - uint32_t* _vpairs_319 = reinterpret_cast(&_vld_319); + uint2 _vld_351; + _vld_351 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (3 ^ qq_1))) + 0); + uint32_t* _vpairs_351 = reinterpret_cast(&_vld_351); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_1[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_319[_pair]) << 16); - (&xv_1[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_319[_pair]) & 0xffff0000u); + (&xv_1[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_351[_pair]) << 16); + (&xv_1[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_351[_pair]) & 0xffff0000u); } } { @@ -4719,65 +4958,65 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 r2_1 = make_float2(rv_1[0], rv_1[0]); float y_1[16]; float p_1[16]; - float2 _f32x2_mul_a_320 = make_float2(xv_1[0], xv_1[1]); + float2 _f32x2_mul_a_352 = make_float2(xv_1[0], xv_1[1]); float2 y2_16; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_16) : "l"(*(const unsigned long long*)&_f32x2_mul_a_320), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_16) : "l"(*(const unsigned long long*)&_f32x2_mul_a_352), "l"(*(const unsigned long long*)&r2_1)); y_1[0] = y2_16.x; y_1[1] = y2_16.y; float2 p2_29 = fma_f32x2_rn_noftz(make_float2(g_1[0], g_1[1]), y2_16, nz2_1); p_1[0] = p2_29.x; p_1[1] = p2_29.y; - float2 _f32x2_mul_a_321 = make_float2(xv_1[2], xv_1[3]); + float2 _f32x2_mul_a_353 = make_float2(xv_1[2], xv_1[3]); float2 y2_17; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_17) : "l"(*(const unsigned long long*)&_f32x2_mul_a_321), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_17) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&r2_1)); y_1[2] = y2_17.x; y_1[3] = y2_17.y; float2 p2_30 = fma_f32x2_rn_noftz(make_float2(g_1[2], g_1[3]), y2_17, nz2_1); p_1[2] = p2_30.x; p_1[3] = p2_30.y; - float2 _f32x2_mul_a_322 = make_float2(xv_1[4], xv_1[5]); + float2 _f32x2_mul_a_354 = make_float2(xv_1[4], xv_1[5]); float2 y2_18; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_18) : "l"(*(const unsigned long long*)&_f32x2_mul_a_322), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_18) : "l"(*(const unsigned long long*)&_f32x2_mul_a_354), "l"(*(const unsigned long long*)&r2_1)); y_1[4] = y2_18.x; y_1[5] = y2_18.y; float2 p2_31 = fma_f32x2_rn_noftz(make_float2(g_1[4], g_1[5]), y2_18, nz2_1); p_1[4] = p2_31.x; p_1[5] = p2_31.y; - float2 _f32x2_mul_a_323 = make_float2(xv_1[6], xv_1[7]); + float2 _f32x2_mul_a_355 = make_float2(xv_1[6], xv_1[7]); float2 y2_19; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_19) : "l"(*(const unsigned long long*)&_f32x2_mul_a_323), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_19) : "l"(*(const unsigned long long*)&_f32x2_mul_a_355), "l"(*(const unsigned long long*)&r2_1)); y_1[6] = y2_19.x; y_1[7] = y2_19.y; float2 p2_32 = fma_f32x2_rn_noftz(make_float2(g_1[6], g_1[7]), y2_19, nz2_1); p_1[6] = p2_32.x; p_1[7] = p2_32.y; - float2 _f32x2_mul_a_324 = make_float2(xv_1[8], xv_1[9]); + float2 _f32x2_mul_a_356 = make_float2(xv_1[8], xv_1[9]); float2 y2_20; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_324), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_356), "l"(*(const unsigned long long*)&r2_1)); y_1[8] = y2_20.x; y_1[9] = y2_20.y; float2 p2_33 = fma_f32x2_rn_noftz(make_float2(g_1[8], g_1[9]), y2_20, nz2_1); p_1[8] = p2_33.x; p_1[9] = p2_33.y; - float2 _f32x2_mul_a_325 = make_float2(xv_1[10], xv_1[11]); + float2 _f32x2_mul_a_357 = make_float2(xv_1[10], xv_1[11]); float2 y2_21; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_325), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_357), "l"(*(const unsigned long long*)&r2_1)); y_1[10] = y2_21.x; y_1[11] = y2_21.y; float2 p2_34 = fma_f32x2_rn_noftz(make_float2(g_1[10], g_1[11]), y2_21, nz2_1); p_1[10] = p2_34.x; p_1[11] = p2_34.y; - float2 _f32x2_mul_a_326 = make_float2(xv_1[12], xv_1[13]); + float2 _f32x2_mul_a_358 = make_float2(xv_1[12], xv_1[13]); float2 y2_22; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_326), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_358), "l"(*(const unsigned long long*)&r2_1)); y_1[12] = y2_22.x; y_1[13] = y2_22.y; float2 p2_35 = fma_f32x2_rn_noftz(make_float2(g_1[12], g_1[13]), y2_22, nz2_1); p_1[12] = p2_35.x; p_1[13] = p2_35.y; - float2 _f32x2_mul_a_327 = make_float2(xv_1[14], xv_1[15]); + float2 _f32x2_mul_a_359 = make_float2(xv_1[14], xv_1[15]); float2 y2_23; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_327), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_359), "l"(*(const unsigned long long*)&r2_1)); y_1[14] = y2_23.x; y_1[15] = y2_23.y; float2 p2_36 = fma_f32x2_rn_noftz(make_float2(g_1[14], g_1[15]), y2_23, nz2_1); @@ -4794,35 +5033,35 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 e2_12 = add_f32x2_noftz(a2_5, c2_5); e_1[2] = e2_12.x; e_1[3] = e2_12.y; - float _shfl_xor_sync_329 = __shfl_xor_sync(0xFFFFFFFF, e_1[0], 1); - float _add_328; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(e_1[0]), "f"(_shfl_xor_sync_329)); - e_1[0] = _add_328; - float _shfl_xor_sync_331 = __shfl_xor_sync(0xFFFFFFFF, e_1[1], 1); - float _add_330; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(e_1[1]), "f"(_shfl_xor_sync_331)); - e_1[1] = _add_330; - float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, e_1[2], 1); - float _add_332; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_332) : "f"(e_1[2]), "f"(_shfl_xor_sync_333)); - e_1[2] = _add_332; - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, e_1[3], 1); - float _add_334; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(e_1[3]), "f"(_shfl_xor_sync_335)); - e_1[3] = _add_334; + float _shfl_xor_sync_361 = __shfl_xor_sync(0xFFFFFFFF, e_1[0], 1); + float _add_360; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_360) : "f"(e_1[0]), "f"(_shfl_xor_sync_361)); + e_1[0] = _add_360; + float _shfl_xor_sync_363 = __shfl_xor_sync(0xFFFFFFFF, e_1[1], 1); + float _add_362; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_362) : "f"(e_1[1]), "f"(_shfl_xor_sync_363)); + e_1[1] = _add_362; + float _shfl_xor_sync_365 = __shfl_xor_sync(0xFFFFFFFF, e_1[2], 1); + float _add_364; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_364) : "f"(e_1[2]), "f"(_shfl_xor_sync_365)); + e_1[2] = _add_364; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, e_1[3], 1); + float _add_366; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_366) : "f"(e_1[3]), "f"(_shfl_xor_sync_367)); + e_1[3] = _add_366; float2 f2_1 = add_f32x2_noftz(make_float2(e_1[0], e_1[1]), make_float2(e_1[2], e_1[3])); float sq_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1) : "f"(f2_1.x), "f"(f2_1.y)); - float _shfl_sync_338 = __shfl_sync(0xFFFFFFFF, sq_1, 0, 8); - float _shfl_sync_339 = __shfl_sync(0xFFFFFFFF, sq_1, 2, 8); - float _add_337; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_337) : "f"(_shfl_sync_338), "f"(_shfl_sync_339)); - float _shfl_sync_340 = __shfl_sync(0xFFFFFFFF, sq_1, 4, 8); - float _add_336; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_336) : "f"(_add_337), "f"(_shfl_sync_340)); - float _shfl_sync_341 = __shfl_sync(0xFFFFFFFF, sq_1, 6, 8); + float _shfl_sync_370 = __shfl_sync(0xFFFFFFFF, sq_1, 0, 8); + float _shfl_sync_371 = __shfl_sync(0xFFFFFFFF, sq_1, 2, 8); + float _add_369; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_369) : "f"(_shfl_sync_370), "f"(_shfl_sync_371)); + float _shfl_sync_372 = __shfl_sync(0xFFFFFFFF, sq_1, 4, 8); + float _add_368; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_368) : "f"(_add_369), "f"(_shfl_sync_372)); + float _shfl_sync_373 = __shfl_sync(0xFFFFFFFF, sq_1, 6, 8); float dot_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_336), "f"(_shfl_sync_341)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_368), "f"(_shfl_sync_373)); float2 d2_62 = make_float2(dot_1, dot_1); float o_8[4]; float2 t2_16 = fma_f32x2_rn_noftz(make_float2(y_1[0], y_1[1]), d2_62, nz2_1); @@ -4906,8 +5145,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM asm volatile("{ .reg .pred p_; setp.ne.b32 p_, %3, 0; @p_ st.global.v2.b32 [%0], {%1, %2}; }" :: "l"(dxp_1 + (orow_1 + (long long)(8 * (3 ^ qq_1)))), "r"(ob_11[(0) + 0]), "r"(ob_11[(0) + 1]), "r"((unsigned int)(live_1)) : "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_342 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_342), "f"(tot_1471) : "memory"); + uint32_t _addr_374 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_374), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4936,9 +5175,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_8 = offset_1488 + _tmem_load_25[0]; if (t_33 < clen) { { - auto _gst_343 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_343)); - *(reinterpret_cast(_gst_343) + (0)) = dg_8; + auto _gst_375 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_375)); + *(reinterpret_cast(_gst_375) + (0)) = dg_8; } } int t_34 = 16 + 8 * th + 1; @@ -4946,9 +5185,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_9 = offset_1488 + _tmem_load_25[1]; if (t_34 < clen) { { - auto _gst_344 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_344)); - *(reinterpret_cast(_gst_344) + (0)) = dg_9; + auto _gst_376 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_376)); + *(reinterpret_cast(_gst_376) + (0)) = dg_9; } } int t_35 = 16 + 8 * th + 2; @@ -4956,9 +5195,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_10 = offset_1488 + _tmem_load_25[2]; if (t_35 < clen) { { - auto _gst_345 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_10; + auto _gst_377 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_10; } } int t_36 = 16 + 8 * th + 3; @@ -4966,9 +5205,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_11 = offset_1488 + _tmem_load_25[3]; if (t_36 < clen) { { - auto _gst_346 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_11; + auto _gst_378 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_11; } } int t_37 = 16 + 8 * th + 4; @@ -4976,9 +5215,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_12 = offset_1488 + _tmem_load_25[4]; if (t_37 < clen) { { - auto _gst_347 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_12; + auto _gst_379 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_12; } } int t_38 = 16 + 8 * th + 5; @@ -4986,9 +5225,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_13 = offset_1488 + _tmem_load_25[5]; if (t_38 < clen) { { - auto _gst_348 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_13; + auto _gst_380 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_13; } } int t_39 = 16 + 8 * th + 6; @@ -4996,9 +5235,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_14 = offset_1488 + _tmem_load_25[6]; if (t_39 < clen) { { - auto _gst_349 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_14; + auto _gst_381 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_14; } } int t_40 = 16 + 8 * th + 7; @@ -5006,15 +5245,15 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_15 = offset_1488 + _tmem_load_25[7]; if (t_40 < clen) { { - auto _gst_350 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_15; + auto _gst_382 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_1486 + t_lo_1487; { - uint32_t _addr_351 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_351), "f"(off_acc[0]) : "memory"); + uint32_t _addr_383 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_383), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -5068,7 +5307,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[7]; - float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, (dk2b_17 - _tmem_load_28[7]) * k_t_17) + dgf_17; + float qe_23 = q_t_1_24 * e_dg_17; + float kp_35 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_35; + float kbe_23 = (float)kb_23 * e_dg_17; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_35 * e_dg_17) * _tmem_load_27[7]); + float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, __fmaf_rn(dk2b_17 - _tmem_load_28[7], k_t_17, cr_24)) + dgf_17; s_run_1524[0] = s_run_1524[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5100,7 +5344,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[6]; - float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, (dk2b_18 - _tmem_load_28[6]) * k_t_18) + dgf_18; + float qe_24 = q_t_1_25 * e_dg_18; + float kp_36 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_36; + float kbe_24 = (float)kb_24 * e_dg_18; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_36 * e_dg_18) * _tmem_load_27[6]); + float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, __fmaf_rn(dk2b_18 - _tmem_load_28[6], k_t_18, cr_25)) + dgf_18; s_run_1524[0] = s_run_1524[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5132,7 +5381,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[5]; - float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, (dk2b_19 - _tmem_load_28[5]) * k_t_19) + dgf_19; + float qe_25 = q_t_1_26 * e_dg_19; + float kp_37 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_37; + float kbe_25 = (float)kb_25 * e_dg_19; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_37 * e_dg_19) * _tmem_load_27[5]); + float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, __fmaf_rn(dk2b_19 - _tmem_load_28[5], k_t_19, cr_26)) + dgf_19; s_run_1524[0] = s_run_1524[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5164,7 +5418,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[4]; - float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, (dk2b_20 - _tmem_load_28[4]) * k_t_20) + dgf_20; + float qe_26 = q_t_1_27 * e_dg_20; + float kp_38 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_38; + float kbe_26 = (float)kb_26 * e_dg_20; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_38 * e_dg_20) * _tmem_load_27[4]); + float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, __fmaf_rn(dk2b_20 - _tmem_load_28[4], k_t_20, cr_27)) + dgf_20; s_run_1524[0] = s_run_1524[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5196,7 +5455,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[3]; - float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, (dk2b_21 - _tmem_load_28[3]) * k_t_21) + dgf_21; + float qe_27 = q_t_1_28 * e_dg_21; + float kp_39 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_39; + float kbe_27 = (float)kb_27 * e_dg_21; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_39 * e_dg_21) * _tmem_load_27[3]); + float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, __fmaf_rn(dk2b_21 - _tmem_load_28[3], k_t_21, cr_28)) + dgf_21; s_run_1524[0] = s_run_1524[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5228,7 +5492,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[2]; - float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, (dk2b_22 - _tmem_load_28[2]) * k_t_22) + dgf_22; + float qe_28 = q_t_1_29 * e_dg_22; + float kp_40 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_40; + float kbe_28 = (float)kb_28 * e_dg_22; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_40 * e_dg_22) * _tmem_load_27[2]); + float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, __fmaf_rn(dk2b_22 - _tmem_load_28[2], k_t_22, cr_29)) + dgf_22; s_run_1524[0] = s_run_1524[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5260,7 +5529,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[1]; - float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, (dk2b_23 - _tmem_load_28[1]) * k_t_23) + dgf_23; + float qe_29 = q_t_1_30 * e_dg_23; + float kp_41 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_41; + float kbe_29 = (float)kb_29 * e_dg_23; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_41 * e_dg_23) * _tmem_load_27[1]); + float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, __fmaf_rn(dk2b_23 - _tmem_load_28[1], k_t_23, cr_30)) + dgf_23; s_run_1524[0] = s_run_1524[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5292,7 +5566,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + _tmem_load_28[0]; - float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, (dk2b_24 - _tmem_load_28[0]) * k_t_24) + dgf_24; + float qe_30 = q_t_1_31 * e_dg_24; + float kp_42 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_42; + float kbe_30 = (float)kb_30 * e_dg_24; + float cr_31 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_42 * e_dg_24) * _tmem_load_27[0]); + float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, __fmaf_rn(dk2b_24 - _tmem_load_28[0], k_t_24, cr_31)) + dgf_24; s_run_1524[0] = s_run_1524[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5307,31 +5586,31 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; float send_13 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_352 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_352; + float _shfl_xor_sync_384 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_384; float send_14 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_353 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_353; + float _shfl_xor_sync_385 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_385; float send_15 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_354 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_354; + float _shfl_xor_sync_386 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_386; float send_16 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_355 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_355; + float _shfl_xor_sync_387 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_387; float v2_1702[2]; float send_17 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_356 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_356; + float _shfl_xor_sync_388 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_388; float send_18 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_357 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_357; + float _shfl_xor_sync_389 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_389; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_358 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_358; - float _shfl_xor_sync_359 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_359; - float _shfl_xor_sync_360 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_360; + float _shfl_xor_sync_390 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_390; + float _shfl_xor_sync_391 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_391; + float _shfl_xor_sync_392 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_392; int ew_3 = 98304 - 32768 * th >> 2; float nz_2 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); float2 nz2_2 = make_float2(nz_2, nz_2); @@ -5341,8 +5620,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int qq_2 = lane_1 >> 1 & 3; int j_2 = qd + 4 * tb_2; bool live_2 = 8 * th + j_2 < clen; - int _min_361 = ((8 * th + j_2) < (clen - 1) ? (8 * th + j_2) : (clen - 1)); - long long row_2 = ext0 + (long long)_min_361; + int _min_393 = ((8 * th + j_2) < (clen - 1) ? (8 * th + j_2) : (clen - 1)); + long long row_2 = ext0 + (long long)_min_393; bool is_k_2 = tk_2 == 1; __nv_bfloat16* xr_2 = ((is_k_2) ? (k_raw) : (q_raw)); int xts_2 = ((is_k_2) ? k_raw_row_stride : q_raw_row_stride); @@ -5368,43 +5647,43 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM : "=r"(*reinterpret_cast(&g_2[12])), "=r"(*reinterpret_cast(&g_2[(12) + 1])), "=r"(*reinterpret_cast(&g_2[(12) + 2])), "=r"(*reinterpret_cast(&g_2[(12) + 3])) : "r"(pool_f32_addr + (unsigned int)(4 * (sb_2 + 8 * (3 ^ qq_2))))); { - uint2 _vld_362; - _vld_362 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (0 ^ qq_2))) + 0); - uint32_t* _vpairs_362 = reinterpret_cast(&_vld_362); + uint2 _vld_394; + _vld_394 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (0 ^ qq_2))) + 0); + uint32_t* _vpairs_394 = reinterpret_cast(&_vld_394); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_362[_pair]) << 16); - (&xv_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_362[_pair]) & 0xffff0000u); + (&xv_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_394[_pair]) << 16); + (&xv_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_394[_pair]) & 0xffff0000u); } } { - uint2 _vld_363; - _vld_363 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (1 ^ qq_2))) + 0); - uint32_t* _vpairs_363 = reinterpret_cast(&_vld_363); + uint2 _vld_395; + _vld_395 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (1 ^ qq_2))) + 0); + uint32_t* _vpairs_395 = reinterpret_cast(&_vld_395); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_2[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_363[_pair]) << 16); - (&xv_2[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_363[_pair]) & 0xffff0000u); + (&xv_2[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_395[_pair]) << 16); + (&xv_2[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_395[_pair]) & 0xffff0000u); } } { - uint2 _vld_364; - _vld_364 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (2 ^ qq_2))) + 0); - uint32_t* _vpairs_364 = reinterpret_cast(&_vld_364); + uint2 _vld_396; + _vld_396 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (2 ^ qq_2))) + 0); + uint32_t* _vpairs_396 = reinterpret_cast(&_vld_396); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_2[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_364[_pair]) << 16); - (&xv_2[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_364[_pair]) & 0xffff0000u); + (&xv_2[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_396[_pair]) << 16); + (&xv_2[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_396[_pair]) & 0xffff0000u); } } { - uint2 _vld_365; - _vld_365 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (3 ^ qq_2))) + 0); - uint32_t* _vpairs_365 = reinterpret_cast(&_vld_365); + uint2 _vld_397; + _vld_397 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (3 ^ qq_2))) + 0); + uint32_t* _vpairs_397 = reinterpret_cast(&_vld_397); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_2[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_365[_pair]) << 16); - (&xv_2[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_365[_pair]) & 0xffff0000u); + (&xv_2[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_397[_pair]) << 16); + (&xv_2[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_397[_pair]) & 0xffff0000u); } } { @@ -5413,65 +5692,65 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 r2_2 = make_float2(rv_2[0], rv_2[0]); float y_2[16]; float p_2[16]; - float2 _f32x2_mul_a_366 = make_float2(xv_2[0], xv_2[1]); + float2 _f32x2_mul_a_398 = make_float2(xv_2[0], xv_2[1]); float2 y2_24; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_24) : "l"(*(const unsigned long long*)&_f32x2_mul_a_366), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_24) : "l"(*(const unsigned long long*)&_f32x2_mul_a_398), "l"(*(const unsigned long long*)&r2_2)); y_2[0] = y2_24.x; y_2[1] = y2_24.y; float2 p2_37 = fma_f32x2_rn_noftz(make_float2(g_2[0], g_2[1]), y2_24, nz2_2); p_2[0] = p2_37.x; p_2[1] = p2_37.y; - float2 _f32x2_mul_a_367 = make_float2(xv_2[2], xv_2[3]); + float2 _f32x2_mul_a_399 = make_float2(xv_2[2], xv_2[3]); float2 y2_25; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_25) : "l"(*(const unsigned long long*)&_f32x2_mul_a_367), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_25) : "l"(*(const unsigned long long*)&_f32x2_mul_a_399), "l"(*(const unsigned long long*)&r2_2)); y_2[2] = y2_25.x; y_2[3] = y2_25.y; float2 p2_38 = fma_f32x2_rn_noftz(make_float2(g_2[2], g_2[3]), y2_25, nz2_2); p_2[2] = p2_38.x; p_2[3] = p2_38.y; - float2 _f32x2_mul_a_368 = make_float2(xv_2[4], xv_2[5]); + float2 _f32x2_mul_a_400 = make_float2(xv_2[4], xv_2[5]); float2 y2_26; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_26) : "l"(*(const unsigned long long*)&_f32x2_mul_a_368), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_26) : "l"(*(const unsigned long long*)&_f32x2_mul_a_400), "l"(*(const unsigned long long*)&r2_2)); y_2[4] = y2_26.x; y_2[5] = y2_26.y; float2 p2_39 = fma_f32x2_rn_noftz(make_float2(g_2[4], g_2[5]), y2_26, nz2_2); p_2[4] = p2_39.x; p_2[5] = p2_39.y; - float2 _f32x2_mul_a_369 = make_float2(xv_2[6], xv_2[7]); + float2 _f32x2_mul_a_401 = make_float2(xv_2[6], xv_2[7]); float2 y2_27; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_27) : "l"(*(const unsigned long long*)&_f32x2_mul_a_369), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_27) : "l"(*(const unsigned long long*)&_f32x2_mul_a_401), "l"(*(const unsigned long long*)&r2_2)); y_2[6] = y2_27.x; y_2[7] = y2_27.y; float2 p2_40 = fma_f32x2_rn_noftz(make_float2(g_2[6], g_2[7]), y2_27, nz2_2); p_2[6] = p2_40.x; p_2[7] = p2_40.y; - float2 _f32x2_mul_a_370 = make_float2(xv_2[8], xv_2[9]); + float2 _f32x2_mul_a_402 = make_float2(xv_2[8], xv_2[9]); float2 y2_28; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_28) : "l"(*(const unsigned long long*)&_f32x2_mul_a_370), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_28) : "l"(*(const unsigned long long*)&_f32x2_mul_a_402), "l"(*(const unsigned long long*)&r2_2)); y_2[8] = y2_28.x; y_2[9] = y2_28.y; float2 p2_41 = fma_f32x2_rn_noftz(make_float2(g_2[8], g_2[9]), y2_28, nz2_2); p_2[8] = p2_41.x; p_2[9] = p2_41.y; - float2 _f32x2_mul_a_371 = make_float2(xv_2[10], xv_2[11]); + float2 _f32x2_mul_a_403 = make_float2(xv_2[10], xv_2[11]); float2 y2_29; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_29) : "l"(*(const unsigned long long*)&_f32x2_mul_a_371), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_29) : "l"(*(const unsigned long long*)&_f32x2_mul_a_403), "l"(*(const unsigned long long*)&r2_2)); y_2[10] = y2_29.x; y_2[11] = y2_29.y; float2 p2_42 = fma_f32x2_rn_noftz(make_float2(g_2[10], g_2[11]), y2_29, nz2_2); p_2[10] = p2_42.x; p_2[11] = p2_42.y; - float2 _f32x2_mul_a_372 = make_float2(xv_2[12], xv_2[13]); + float2 _f32x2_mul_a_404 = make_float2(xv_2[12], xv_2[13]); float2 y2_30; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_30) : "l"(*(const unsigned long long*)&_f32x2_mul_a_372), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_30) : "l"(*(const unsigned long long*)&_f32x2_mul_a_404), "l"(*(const unsigned long long*)&r2_2)); y_2[12] = y2_30.x; y_2[13] = y2_30.y; float2 p2_43 = fma_f32x2_rn_noftz(make_float2(g_2[12], g_2[13]), y2_30, nz2_2); p_2[12] = p2_43.x; p_2[13] = p2_43.y; - float2 _f32x2_mul_a_373 = make_float2(xv_2[14], xv_2[15]); + float2 _f32x2_mul_a_405 = make_float2(xv_2[14], xv_2[15]); float2 y2_31; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_31) : "l"(*(const unsigned long long*)&_f32x2_mul_a_373), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_31) : "l"(*(const unsigned long long*)&_f32x2_mul_a_405), "l"(*(const unsigned long long*)&r2_2)); y_2[14] = y2_31.x; y_2[15] = y2_31.y; float2 p2_44 = fma_f32x2_rn_noftz(make_float2(g_2[14], g_2[15]), y2_31, nz2_2); @@ -5488,35 +5767,35 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 e2_14 = add_f32x2_noftz(a2_7, c2_7); e_2[2] = e2_14.x; e_2[3] = e2_14.y; - float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, e_2[0], 1); - float _add_374; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_374) : "f"(e_2[0]), "f"(_shfl_xor_sync_375)); - e_2[0] = _add_374; - float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, e_2[1], 1); - float _add_376; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_376) : "f"(e_2[1]), "f"(_shfl_xor_sync_377)); - e_2[1] = _add_376; - float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, e_2[2], 1); - float _add_378; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_378) : "f"(e_2[2]), "f"(_shfl_xor_sync_379)); - e_2[2] = _add_378; - float _shfl_xor_sync_381 = __shfl_xor_sync(0xFFFFFFFF, e_2[3], 1); - float _add_380; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_380) : "f"(e_2[3]), "f"(_shfl_xor_sync_381)); - e_2[3] = _add_380; + float _shfl_xor_sync_407 = __shfl_xor_sync(0xFFFFFFFF, e_2[0], 1); + float _add_406; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_406) : "f"(e_2[0]), "f"(_shfl_xor_sync_407)); + e_2[0] = _add_406; + float _shfl_xor_sync_409 = __shfl_xor_sync(0xFFFFFFFF, e_2[1], 1); + float _add_408; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_408) : "f"(e_2[1]), "f"(_shfl_xor_sync_409)); + e_2[1] = _add_408; + float _shfl_xor_sync_411 = __shfl_xor_sync(0xFFFFFFFF, e_2[2], 1); + float _add_410; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_410) : "f"(e_2[2]), "f"(_shfl_xor_sync_411)); + e_2[2] = _add_410; + float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, e_2[3], 1); + float _add_412; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_412) : "f"(e_2[3]), "f"(_shfl_xor_sync_413)); + e_2[3] = _add_412; float2 f2_2 = add_f32x2_noftz(make_float2(e_2[0], e_2[1]), make_float2(e_2[2], e_2[3])); float sq_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2) : "f"(f2_2.x), "f"(f2_2.y)); - float _shfl_sync_384 = __shfl_sync(0xFFFFFFFF, sq_2, 0, 8); - float _shfl_sync_385 = __shfl_sync(0xFFFFFFFF, sq_2, 2, 8); - float _add_383; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_383) : "f"(_shfl_sync_384), "f"(_shfl_sync_385)); - float _shfl_sync_386 = __shfl_sync(0xFFFFFFFF, sq_2, 4, 8); - float _add_382; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_382) : "f"(_add_383), "f"(_shfl_sync_386)); - float _shfl_sync_387 = __shfl_sync(0xFFFFFFFF, sq_2, 6, 8); + float _shfl_sync_416 = __shfl_sync(0xFFFFFFFF, sq_2, 0, 8); + float _shfl_sync_417 = __shfl_sync(0xFFFFFFFF, sq_2, 2, 8); + float _add_415; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_415) : "f"(_shfl_sync_416), "f"(_shfl_sync_417)); + float _shfl_sync_418 = __shfl_sync(0xFFFFFFFF, sq_2, 4, 8); + float _add_414; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_414) : "f"(_add_415), "f"(_shfl_sync_418)); + float _shfl_sync_419 = __shfl_sync(0xFFFFFFFF, sq_2, 6, 8); float dot_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_382), "f"(_shfl_sync_387)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_414), "f"(_shfl_sync_419)); float2 d2_63 = make_float2(dot_2, dot_2); float o_12[4]; float2 t2_24 = fma_f32x2_rn_noftz(make_float2(y_2[0], y_2[1]), d2_63, nz2_2); @@ -5600,8 +5879,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM asm volatile("{ .reg .pred p_; setp.ne.b32 p_, %3, 0; @p_ st.global.v2.b32 [%0], {%1, %2}; }" :: "l"(dxp_2 + (orow_2 + (long long)(8 * (3 ^ qq_2)))), "r"(ob_15[(0) + 0]), "r"(ob_15[(0) + 1]), "r"((unsigned int)(live_2)) : "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_388 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_388), "f"(tot_1693) : "memory"); + uint32_t _addr_420 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_420), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5630,9 +5909,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_16 = offset_1710 + _tmem_load_29[0]; if (t_49 < clen) { { - auto _gst_389 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_389)); - *(reinterpret_cast(_gst_389) + (0)) = dg_16; + auto _gst_421 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_421)); + *(reinterpret_cast(_gst_421) + (0)) = dg_16; } } int t_50 = 8 * th + 1; @@ -5640,9 +5919,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_17 = offset_1710 + _tmem_load_29[1]; if (t_50 < clen) { { - auto _gst_390 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_390)); - *(reinterpret_cast(_gst_390) + (0)) = dg_17; + auto _gst_422 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_422)); + *(reinterpret_cast(_gst_422) + (0)) = dg_17; } } int t_51 = 8 * th + 2; @@ -5650,9 +5929,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_18 = offset_1710 + _tmem_load_29[2]; if (t_51 < clen) { { - auto _gst_391 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_391)); - *(reinterpret_cast(_gst_391) + (0)) = dg_18; + auto _gst_423 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_423)); + *(reinterpret_cast(_gst_423) + (0)) = dg_18; } } int t_52 = 8 * th + 3; @@ -5660,9 +5939,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_19 = offset_1710 + _tmem_load_29[3]; if (t_52 < clen) { { - auto _gst_392 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_392)); - *(reinterpret_cast(_gst_392) + (0)) = dg_19; + auto _gst_424 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_424)); + *(reinterpret_cast(_gst_424) + (0)) = dg_19; } } int t_53 = 8 * th + 4; @@ -5670,9 +5949,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_20 = offset_1710 + _tmem_load_29[4]; if (t_53 < clen) { { - auto _gst_393 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_393)); - *(reinterpret_cast(_gst_393) + (0)) = dg_20; + auto _gst_425 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_425)); + *(reinterpret_cast(_gst_425) + (0)) = dg_20; } } int t_54 = 8 * th + 5; @@ -5680,9 +5959,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_21 = offset_1710 + _tmem_load_29[5]; if (t_54 < clen) { { - auto _gst_394 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_394)); - *(reinterpret_cast(_gst_394) + (0)) = dg_21; + auto _gst_426 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_426)); + *(reinterpret_cast(_gst_426) + (0)) = dg_21; } } int t_55 = 8 * th + 6; @@ -5690,9 +5969,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_22 = offset_1710 + _tmem_load_29[6]; if (t_55 < clen) { { - auto _gst_395 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_395)); - *(reinterpret_cast(_gst_395) + (0)) = dg_22; + auto _gst_427 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_427)); + *(reinterpret_cast(_gst_427) + (0)) = dg_22; } } int t_56 = 8 * th + 7; @@ -5700,9 +5979,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_23 = offset_1710 + _tmem_load_29[7]; if (t_56 < clen) { { - auto _gst_396 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_396)); - *(reinterpret_cast(_gst_396) + (0)) = dg_23; + auto _gst_428 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_428)); + *(reinterpret_cast(_gst_428) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1708 + t_lo_1709; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu index 1ccc6d124b9..c37ebe2a51c 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2249,12 +2272,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_3 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * e2_3); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_3, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_3 : __fmaf_rn(dl, _rcp_170, e2_3)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2278,12 +2307,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_4 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * e2_4); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_4, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_4, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_4 : __fmaf_rn(dl_1, _rcp_174, e2_4)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2307,12 +2342,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_5 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * e2_5); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_5, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_5, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_5 : __fmaf_rn(dl_3, _rcp_178, e2_5)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2336,12 +2377,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_6 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * e2_6); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_6, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_6, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_6 : __fmaf_rn(dl_5, _rcp_182, e2_6)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_2 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_1 = approx_exp2(dc2_2.x); float e2_1_1 = approx_exp2(dc2_2.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_190 = approx_rcp(k_t0_2); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_191 = approx_rcp(k_t1_2); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_2.x; pan_641[3] = p2_2.y; int t0_3 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_2 = approx_exp2(dc2_3.x); float e2_1_2 = approx_exp2(dc2_3.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_194 = approx_rcp(k_t0_3); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_195 = approx_rcp(k_t1_3); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_3.x; pan_641[5] = p2_3.y; int t0_4 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(_f2_61, make_float2(g_t0_4, g_t1_4)); float e2_0_3 = approx_exp2(dc2_4.x); float e2_1_3 = approx_exp2(dc2_4.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_198 = approx_rcp(k_t0_4); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_199 = approx_rcp(k_t1_4); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_4.x; pan_641[7] = p2_4.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2505,12 +2576,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_202 = approx_rcp(k_t0_5); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_203 = approx_rcp(k_t1_5); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_5.x; pan_703[1] = p2_5.y; int t0_6 = 8 * th + 2; @@ -2534,12 +2611,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_2.x); float en_1_1 = approx_exp2(dn2_2.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_206 = approx_rcp(k_t0_6); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_207 = approx_rcp(k_t1_6); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_6.x; pan_703[3] = p2_6.y; int t0_7 = 8 * th + 4; @@ -2563,12 +2646,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_3.x); float en_1_2 = approx_exp2(dn2_3.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_210 = approx_rcp(k_t0_7); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_211 = approx_rcp(k_t1_7); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_7.x; pan_703[5] = p2_7.y; int t0_8 = 8 * th + 6; @@ -2592,12 +2681,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_4.x); float en_1_3 = approx_exp2(dn2_4.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_214 = approx_rcp(k_t0_8); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_215 = approx_rcp(k_t1_8); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_8.x; pan_703[7] = p2_8.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2639,12 +2734,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_5.x); float en_1_4 = approx_exp2(dn2_5.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_218 = approx_rcp(k_t0_9); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_219 = approx_rcp(k_t1_9); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_9.x; pan_777[1] = p2_9.y; int t0_10 = 16 + 8 * th + 2; @@ -2668,12 +2769,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_6.x); float en_1_5 = approx_exp2(dn2_6.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_222 = approx_rcp(k_t0_10); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_223 = approx_rcp(k_t1_10); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_10.x; pan_777[3] = p2_10.y; int t0_11 = 16 + 8 * th + 4; @@ -2697,12 +2804,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_7.x); float en_1_6 = approx_exp2(dn2_7.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_226 = approx_rcp(k_t0_11); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_227 = approx_rcp(k_t1_11); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_11.x; pan_777[5] = p2_11.y; int t0_12 = 16 + 8 * th + 6; @@ -2726,12 +2839,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_8 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_8.x); float en_1_7 = approx_exp2(dn2_8.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_230 = approx_rcp(k_t0_12); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_231 = approx_rcp(k_t1_12); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_12.x; pan_777[7] = p2_12.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2802,37 +2921,37 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_4 = approx_exp2(d2_57.x); float e1_4 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_4, e1_4); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_4, e1_4); float2 v2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_4.x; v_562_0[1] = v2_4.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_5 = approx_exp2(d2_58.x); float e1_5 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_5, e1_5); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_5, e1_5); float2 v2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_5.x; v_562_0[3] = v2_5.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_6 = approx_exp2(d2_59.x); float e1_6 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_6, e1_6); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_6, e1_6); float2 v2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_6.x; v_562_0[5] = v2_6.y; float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_7 = approx_exp2(d2_60.x); float e1_7 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_7, e1_7); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_7, e1_7); float2 v2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_7.x; v_562_0[7] = v2_7.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2966,7 +3085,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -2976,14 +3100,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_0 = t_1_0 * qrs; if (t_1_0 < clen) { { - auto _gst_216 = dq_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_1_0; + auto _gst_248 = dq_base + qoff_1_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_1_0; } { - auto _gst_217 = dk_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_1_0; + auto _gst_249 = dk_base + qoff_1_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_1_0; } } int t_1_1 = 32 + 8 * th + 6; @@ -3009,7 +3133,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3019,14 +3148,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_1 = t_1_1 * qrs; if (t_1_1 < clen) { { - auto _gst_218 = dq_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1_1; + auto _gst_250 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1_1; } { - auto _gst_219 = dk_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1_1; + auto _gst_251 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1_1; } } int t_1_2 = 32 + 8 * th + 5; @@ -3052,7 +3181,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_2)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3062,14 +3196,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_2 = t_1_2 * qrs; if (t_1_2 < clen) { { - auto _gst_220 = dq_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1_2; + auto _gst_252 = dq_base + qoff_1_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_1_2; } { - auto _gst_221 = dk_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1_2; + auto _gst_253 = dk_base + qoff_1_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_1_2; } } int t_1_3 = 32 + 8 * th + 4; @@ -3095,7 +3229,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_3)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3105,14 +3244,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_3 = t_1_3 * qrs; if (t_1_3 < clen) { { - auto _gst_222 = dq_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_1_3; + auto _gst_254 = dq_base + qoff_1_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_1_3; } { - auto _gst_223 = dk_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_1_3; + auto _gst_255 = dk_base + qoff_1_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_1_3; } } int t_1_4 = 32 + 8 * th + 3; @@ -3138,7 +3277,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_4)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3148,14 +3292,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_4 = t_1_4 * qrs; if (t_1_4 < clen) { { - auto _gst_224 = dq_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_1_4; + auto _gst_256 = dq_base + qoff_1_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_1_4; } { - auto _gst_225 = dk_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_1_4; + auto _gst_257 = dk_base + qoff_1_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_1_4; } } int t_1_5 = 32 + 8 * th + 2; @@ -3181,7 +3325,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_5)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3191,14 +3340,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_5 = t_1_5 * qrs; if (t_1_5 < clen) { { - auto _gst_226 = dq_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_1_5; + auto _gst_258 = dq_base + qoff_1_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_1_5; } { - auto _gst_227 = dk_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_1_5; + auto _gst_259 = dk_base + qoff_1_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_1_5; } } int t_1_6 = 32 + 8 * th + 1; @@ -3224,7 +3373,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_6)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3234,14 +3388,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_6 = t_1_6 * qrs; if (t_1_6 < clen) { { - auto _gst_228 = dq_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_1_6; + auto _gst_260 = dq_base + qoff_1_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_1_6; } { - auto _gst_229 = dk_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_1_6; + auto _gst_261 = dk_base + qoff_1_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_1_6; } } int t_1_7 = 32 + 8 * th; @@ -3267,7 +3421,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_7)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3277,14 +3436,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_7 = t_1_7 * qrs; if (t_1_7 < clen) { { - auto _gst_230 = dq_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_1_7; + auto _gst_262 = dq_base + qoff_1_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_1_7; } { - auto _gst_231 = dk_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_1_7; + auto _gst_263 = dk_base + qoff_1_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_1_7; } } float tot = s_run[0]; @@ -3293,31 +3452,31 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3377,7 +3536,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; + float qe_7 = q_t_1_8 * e_dg_0; + float kp_19 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_0; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_14[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_0) * _tmem_load_15[7]); + float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_16[7], k_t_0, cr_8)) + dgf_0; s_run_1059[0] = s_run_1059[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3387,14 +3551,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_241 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_273 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_242 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_274 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_2 = 48 + 8 * th + 6; @@ -3420,7 +3584,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[6]; - float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, (dk2b_2 - _tmem_load_16[6]) * k_t_2) + dgf_2; + float qe_8 = q_t_1_9 * e_dg_2; + float kp_20 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_14[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_2) * _tmem_load_15[6]); + float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_16[6], k_t_2, cr_9)) + dgf_2; s_run_1059[0] = s_run_1059[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3430,14 +3599,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_243 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_275 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_244 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_276 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 48 + 8 * th + 5; @@ -3463,7 +3632,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[5]; - float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, (dk2b_3 - _tmem_load_16[5]) * k_t_3) + dgf_3; + float qe_9 = q_t_1_10 * e_dg_3; + float kp_21 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_3; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_14[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_3) * _tmem_load_15[5]); + float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_16[5], k_t_3, cr_10)) + dgf_3; s_run_1059[0] = s_run_1059[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3473,14 +3647,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_245 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_277 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_246 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_278 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 48 + 8 * th + 4; @@ -3506,7 +3680,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[4]; - float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, (dk2b_4 - _tmem_load_16[4]) * k_t_4) + dgf_4; + float qe_10 = q_t_1_11 * e_dg_4; + float kp_22 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_4; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_14[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_4) * _tmem_load_15[4]); + float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_16[4], k_t_4, cr_11)) + dgf_4; s_run_1059[0] = s_run_1059[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3516,14 +3695,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_247 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_279 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_248 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_280 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 48 + 8 * th + 3; @@ -3549,7 +3728,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[3]; - float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, (dk2b_5 - _tmem_load_16[3]) * k_t_5) + dgf_5; + float qe_11 = q_t_1_12 * e_dg_5; + float kp_23 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_5; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_14[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_5) * _tmem_load_15[3]); + float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_16[3], k_t_5, cr_12)) + dgf_5; s_run_1059[0] = s_run_1059[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3559,14 +3743,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_249 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_281 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_250 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_282 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 48 + 8 * th + 2; @@ -3592,7 +3776,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[2]; - float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, (dk2b_6 - _tmem_load_16[2]) * k_t_6) + dgf_6; + float qe_12 = q_t_1_13 * e_dg_6; + float kp_24 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_6; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_14[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_6) * _tmem_load_15[2]); + float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_16[2], k_t_6, cr_13)) + dgf_6; s_run_1059[0] = s_run_1059[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3602,14 +3791,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_251 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_283 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_252 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_284 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 48 + 8 * th + 1; @@ -3635,7 +3824,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[1]; - float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, (dk2b_7 - _tmem_load_16[1]) * k_t_7) + dgf_7; + float qe_13 = q_t_1_14 * e_dg_7; + float kp_25 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_7; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_14[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_7) * _tmem_load_15[1]); + float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_16[1], k_t_7, cr_14)) + dgf_7; s_run_1059[0] = s_run_1059[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3645,14 +3839,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_253 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_285 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_254 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_286 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_7; } } int t_8 = 48 + 8 * th; @@ -3678,7 +3872,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_16[0]; - float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, (dk2b_8 - _tmem_load_16[0]) * k_t_8) + dgf_8; + float qe_14 = q_t_1_15 * e_dg_8; + float kp_26 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_8; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_14[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_8) * _tmem_load_15[0]); + float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_16[0], k_t_8, cr_15)) + dgf_8; s_run_1059[0] = s_run_1059[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3688,14 +3887,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_255 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_287 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_256 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_288 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_8; } } float tot_1228 = s_run_1059[0]; @@ -3704,36 +3903,36 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int b2_1231 = lane_1 >> 2 & 1; float v4_1232[4]; float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_289; float send_2 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_290; float send_3 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_291; float send_4 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_292; float v2_1237[2]; float send_5 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_293; float send_6 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); + v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_294; float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); + float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); + v1_1241 = v1_1241 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); + v1_1241 = v1_1241 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_1228) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_1228) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3762,9 +3961,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_0 = offset + _tmem_load_17[0]; if (t_9 < clen) { { - auto _gst_267 = dg_base + goff_1_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_1_0; + auto _gst_299 = dg_base + goff_1_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_1_0; } } int t_10 = 48 + 8 * th + 1; @@ -3772,9 +3971,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_1 = offset + _tmem_load_17[1]; if (t_10 < clen) { { - auto _gst_268 = dg_base + goff_1_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1_1; + auto _gst_300 = dg_base + goff_1_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1_1; } } int t_11 = 48 + 8 * th + 2; @@ -3782,9 +3981,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_2 = offset + _tmem_load_17[2]; if (t_11 < clen) { { - auto _gst_269 = dg_base + goff_1_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_1_2; + auto _gst_301 = dg_base + goff_1_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_1_2; } } int t_12 = 48 + 8 * th + 3; @@ -3792,9 +3991,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_3 = offset + _tmem_load_17[3]; if (t_12 < clen) { { - auto _gst_270 = dg_base + goff_1_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_1_3; + auto _gst_302 = dg_base + goff_1_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_1_3; } } int t_13 = 48 + 8 * th + 4; @@ -3802,9 +4001,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_4 = offset + _tmem_load_17[4]; if (t_13 < clen) { { - auto _gst_271 = dg_base + goff_1_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_1_4; + auto _gst_303 = dg_base + goff_1_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_1_4; } } int t_14 = 48 + 8 * th + 5; @@ -3812,9 +4011,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_5 = offset + _tmem_load_17[5]; if (t_14 < clen) { { - auto _gst_272 = dg_base + goff_1_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_1_5; + auto _gst_304 = dg_base + goff_1_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_1_5; } } int t_15 = 48 + 8 * th + 6; @@ -3822,9 +4021,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_6 = offset + _tmem_load_17[6]; if (t_15 < clen) { { - auto _gst_273 = dg_base + goff_1_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_1_6; + auto _gst_305 = dg_base + goff_1_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_1_6; } } int t_16 = 48 + 8 * th + 7; @@ -3832,9 +4031,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_7 = offset + _tmem_load_17[7]; if (t_16 < clen) { { - auto _gst_274 = dg_base + goff_1_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_1_7; + auto _gst_306 = dg_base + goff_1_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_1_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3852,9 +4051,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_0 = offset_1250 + _tmem_load_19[0]; if (t_17 < clen) { { - auto _gst_275 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_0; + auto _gst_307 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_0; } } int t_18 = 32 + 8 * th + 1; @@ -3862,9 +4061,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1 = offset_1250 + _tmem_load_19[1]; if (t_18 < clen) { { - auto _gst_276 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_1; + auto _gst_308 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_1; } } int t_19 = 32 + 8 * th + 2; @@ -3872,9 +4071,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_2 = offset_1250 + _tmem_load_19[2]; if (t_19 < clen) { { - auto _gst_277 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_2; + auto _gst_309 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_2; } } int t_20 = 32 + 8 * th + 3; @@ -3882,9 +4081,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_3 = offset_1250 + _tmem_load_19[3]; if (t_20 < clen) { { - auto _gst_278 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_3; + auto _gst_310 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_3; } } int t_21 = 32 + 8 * th + 4; @@ -3892,9 +4091,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_4 = offset_1250 + _tmem_load_19[4]; if (t_21 < clen) { { - auto _gst_279 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_4; + auto _gst_311 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_4; } } int t_22 = 32 + 8 * th + 5; @@ -3902,9 +4101,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_5 = offset_1250 + _tmem_load_19[5]; if (t_22 < clen) { { - auto _gst_280 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_5; + auto _gst_312 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_5; } } int t_23 = 32 + 8 * th + 6; @@ -3912,9 +4111,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_6 = offset_1250 + _tmem_load_19[6]; if (t_23 < clen) { { - auto _gst_281 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_6; + auto _gst_313 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_6; } } int t_24 = 32 + 8 * th + 7; @@ -3922,15 +4121,15 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_7 = offset_1250 + _tmem_load_19[7]; if (t_24 < clen) { { - auto _gst_282 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_7; + auto _gst_314 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi_1248 + t_lo_1249; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3990,7 +4189,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[7]; - float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, (dk2b_9 - _tmem_load_24[7]) * k_t_9) + dgf_9; + float qe_15 = q_t_1_16 * e_dg_9; + float kp_27 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_9; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_9) * _tmem_load_23[7]); + float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_24[7], k_t_9, cr_16)) + dgf_9; s_run_1302[0] = s_run_1302[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4000,14 +4204,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_9 = t_25 * qrs; if (t_25 < clen) { { - auto _gst_284 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_316 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_285 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_317 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_26 = 16 + 8 * th + 6; @@ -4033,7 +4237,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[6]; - float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, (dk2b_10 - _tmem_load_24[6]) * k_t_10) + dgf_10; + float qe_16 = q_t_1_17 * e_dg_10; + float kp_28 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_10; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_10) * _tmem_load_23[6]); + float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_24[6], k_t_10, cr_17)) + dgf_10; s_run_1302[0] = s_run_1302[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4043,14 +4252,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_10 = t_26 * qrs; if (t_26 < clen) { { - auto _gst_286 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_318 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_287 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_319 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_27 = 16 + 8 * th + 5; @@ -4076,7 +4285,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[5]; - float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, (dk2b_11 - _tmem_load_24[5]) * k_t_11) + dgf_11; + float qe_17 = q_t_1_18 * e_dg_11; + float kp_29 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_11; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_11) * _tmem_load_23[5]); + float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_24[5], k_t_11, cr_18)) + dgf_11; s_run_1302[0] = s_run_1302[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4086,14 +4300,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_11 = t_27 * qrs; if (t_27 < clen) { { - auto _gst_288 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_320 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_289 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_321 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_28 = 16 + 8 * th + 4; @@ -4119,7 +4333,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[4]; - float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, (dk2b_12 - _tmem_load_24[4]) * k_t_12) + dgf_12; + float qe_18 = q_t_1_19 * e_dg_12; + float kp_30 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_12; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_12) * _tmem_load_23[4]); + float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_24[4], k_t_12, cr_19)) + dgf_12; s_run_1302[0] = s_run_1302[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4129,14 +4348,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_12 = t_28 * qrs; if (t_28 < clen) { { - auto _gst_290 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_322 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_291 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_323 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_29 = 16 + 8 * th + 3; @@ -4162,7 +4381,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[3]; - float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, (dk2b_13 - _tmem_load_24[3]) * k_t_13) + dgf_13; + float qe_19 = q_t_1_20 * e_dg_13; + float kp_31 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_13; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_13) * _tmem_load_23[3]); + float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_24[3], k_t_13, cr_20)) + dgf_13; s_run_1302[0] = s_run_1302[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4172,14 +4396,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_13 = t_29 * qrs; if (t_29 < clen) { { - auto _gst_292 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_324 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_293 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_325 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_30 = 16 + 8 * th + 2; @@ -4205,7 +4429,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[2]; - float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, (dk2b_14 - _tmem_load_24[2]) * k_t_14) + dgf_14; + float qe_20 = q_t_1_21 * e_dg_14; + float kp_32 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_14; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_14) * _tmem_load_23[2]); + float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_24[2], k_t_14, cr_21)) + dgf_14; s_run_1302[0] = s_run_1302[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4215,14 +4444,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_14 = t_30 * qrs; if (t_30 < clen) { { - auto _gst_294 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_326 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_295 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_327 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_31 = 16 + 8 * th + 1; @@ -4248,7 +4477,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[1]; - float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, (dk2b_15 - _tmem_load_24[1]) * k_t_15) + dgf_15; + float qe_21 = q_t_1_22 * e_dg_15; + float kp_33 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_15; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_15) * _tmem_load_23[1]); + float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_24[1], k_t_15, cr_22)) + dgf_15; s_run_1302[0] = s_run_1302[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4258,14 +4492,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_15 = t_31 * qrs; if (t_31 < clen) { { - auto _gst_296 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_328 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_297 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_329 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_15; } } int t_32 = 16 + 8 * th; @@ -4291,7 +4525,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_24[0]; - float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, (dk2b_16 - _tmem_load_24[0]) * k_t_16) + dgf_16; + float qe_22 = q_t_1_23 * e_dg_16; + float kp_34 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_16; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_16) * _tmem_load_23[0]); + float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, __fmaf_rn(dk2b_16 - _tmem_load_24[0], k_t_16, cr_23)) + dgf_16; s_run_1302[0] = s_run_1302[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4301,14 +4540,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_298 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_330 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_299 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_331 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_16; } } float tot_1471 = s_run_1302[0]; @@ -4317,37 +4556,37 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; float send_7 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_332; float send_8 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_333; float send_9 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_334; float send_10 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_335; float v2_1480[2]; float send_11 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_336; float send_12 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_337; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1471) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4376,9 +4615,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_8 = offset_1488 + _tmem_load_25[0]; if (t_33 < clen) { { - auto _gst_310 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_8; + auto _gst_342 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_8; } } int t_34 = 16 + 8 * th + 1; @@ -4386,9 +4625,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_9 = offset_1488 + _tmem_load_25[1]; if (t_34 < clen) { { - auto _gst_311 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_9; + auto _gst_343 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_9; } } int t_35 = 16 + 8 * th + 2; @@ -4396,9 +4635,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_10 = offset_1488 + _tmem_load_25[2]; if (t_35 < clen) { { - auto _gst_312 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_10; + auto _gst_344 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_10; } } int t_36 = 16 + 8 * th + 3; @@ -4406,9 +4645,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_11 = offset_1488 + _tmem_load_25[3]; if (t_36 < clen) { { - auto _gst_313 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_11; + auto _gst_345 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_11; } } int t_37 = 16 + 8 * th + 4; @@ -4416,9 +4655,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_12 = offset_1488 + _tmem_load_25[4]; if (t_37 < clen) { { - auto _gst_314 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_12; + auto _gst_346 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_12; } } int t_38 = 16 + 8 * th + 5; @@ -4426,9 +4665,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_13 = offset_1488 + _tmem_load_25[5]; if (t_38 < clen) { { - auto _gst_315 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_13; + auto _gst_347 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_13; } } int t_39 = 16 + 8 * th + 6; @@ -4436,9 +4675,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_14 = offset_1488 + _tmem_load_25[6]; if (t_39 < clen) { { - auto _gst_316 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_14; + auto _gst_348 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_14; } } int t_40 = 16 + 8 * th + 7; @@ -4446,15 +4685,15 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_15 = offset_1488 + _tmem_load_25[7]; if (t_40 < clen) { { - auto _gst_317 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_15; + auto _gst_349 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_1486 + t_lo_1487; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4508,7 +4747,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[7]; - float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, (dk2b_17 - _tmem_load_28[7]) * k_t_17) + dgf_17; + float qe_23 = q_t_1_24 * e_dg_17; + float kp_35 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_35; + float kbe_23 = (float)kb_23 * e_dg_17; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_35 * e_dg_17) * _tmem_load_27[7]); + float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, __fmaf_rn(dk2b_17 - _tmem_load_28[7], k_t_17, cr_24)) + dgf_17; s_run_1524[0] = s_run_1524[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4518,14 +4762,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_17 = t_41 * qrs; if (t_41 < clen) { { - auto _gst_319 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_351 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_320 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_352 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_42 = 8 * th + 6; @@ -4551,7 +4795,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[6]; - float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, (dk2b_18 - _tmem_load_28[6]) * k_t_18) + dgf_18; + float qe_24 = q_t_1_25 * e_dg_18; + float kp_36 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_36; + float kbe_24 = (float)kb_24 * e_dg_18; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_36 * e_dg_18) * _tmem_load_27[6]); + float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, __fmaf_rn(dk2b_18 - _tmem_load_28[6], k_t_18, cr_25)) + dgf_18; s_run_1524[0] = s_run_1524[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4561,14 +4810,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_18 = t_42 * qrs; if (t_42 < clen) { { - auto _gst_321 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_353 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_322 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_354 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_43 = 8 * th + 5; @@ -4594,7 +4843,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[5]; - float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, (dk2b_19 - _tmem_load_28[5]) * k_t_19) + dgf_19; + float qe_25 = q_t_1_26 * e_dg_19; + float kp_37 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_37; + float kbe_25 = (float)kb_25 * e_dg_19; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_37 * e_dg_19) * _tmem_load_27[5]); + float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, __fmaf_rn(dk2b_19 - _tmem_load_28[5], k_t_19, cr_26)) + dgf_19; s_run_1524[0] = s_run_1524[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4604,14 +4858,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_19 = t_43 * qrs; if (t_43 < clen) { { - auto _gst_323 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_355 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_324 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_356 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_44 = 8 * th + 4; @@ -4637,7 +4891,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[4]; - float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, (dk2b_20 - _tmem_load_28[4]) * k_t_20) + dgf_20; + float qe_26 = q_t_1_27 * e_dg_20; + float kp_38 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_38; + float kbe_26 = (float)kb_26 * e_dg_20; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_38 * e_dg_20) * _tmem_load_27[4]); + float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, __fmaf_rn(dk2b_20 - _tmem_load_28[4], k_t_20, cr_27)) + dgf_20; s_run_1524[0] = s_run_1524[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4647,14 +4906,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_20 = t_44 * qrs; if (t_44 < clen) { { - auto _gst_325 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_357 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_326 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_358 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_45 = 8 * th + 3; @@ -4680,7 +4939,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[3]; - float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, (dk2b_21 - _tmem_load_28[3]) * k_t_21) + dgf_21; + float qe_27 = q_t_1_28 * e_dg_21; + float kp_39 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_39; + float kbe_27 = (float)kb_27 * e_dg_21; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_39 * e_dg_21) * _tmem_load_27[3]); + float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, __fmaf_rn(dk2b_21 - _tmem_load_28[3], k_t_21, cr_28)) + dgf_21; s_run_1524[0] = s_run_1524[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4690,14 +4954,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_21 = t_45 * qrs; if (t_45 < clen) { { - auto _gst_327 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_359 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_328 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_360 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_46 = 8 * th + 2; @@ -4723,7 +4987,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[2]; - float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, (dk2b_22 - _tmem_load_28[2]) * k_t_22) + dgf_22; + float qe_28 = q_t_1_29 * e_dg_22; + float kp_40 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_40; + float kbe_28 = (float)kb_28 * e_dg_22; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_40 * e_dg_22) * _tmem_load_27[2]); + float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, __fmaf_rn(dk2b_22 - _tmem_load_28[2], k_t_22, cr_29)) + dgf_22; s_run_1524[0] = s_run_1524[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4733,14 +5002,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_22 = t_46 * qrs; if (t_46 < clen) { { - auto _gst_329 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_361 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_330 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_362 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_47 = 8 * th + 1; @@ -4766,7 +5035,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[1]; - float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, (dk2b_23 - _tmem_load_28[1]) * k_t_23) + dgf_23; + float qe_29 = q_t_1_30 * e_dg_23; + float kp_41 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_41; + float kbe_29 = (float)kb_29 * e_dg_23; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_41 * e_dg_23) * _tmem_load_27[1]); + float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, __fmaf_rn(dk2b_23 - _tmem_load_28[1], k_t_23, cr_30)) + dgf_23; s_run_1524[0] = s_run_1524[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4776,14 +5050,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_23 = t_47 * qrs; if (t_47 < clen) { { - auto _gst_331 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_363 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_332 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_364 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_23; } } int t_48 = 8 * th; @@ -4809,7 +5083,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + _tmem_load_28[0]; - float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, (dk2b_24 - _tmem_load_28[0]) * k_t_24) + dgf_24; + float qe_30 = q_t_1_31 * e_dg_24; + float kp_42 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_42; + float kbe_30 = (float)kb_30 * e_dg_24; + float cr_31 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_42 * e_dg_24) * _tmem_load_27[0]); + float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, __fmaf_rn(dk2b_24 - _tmem_load_28[0], k_t_24, cr_31)) + dgf_24; s_run_1524[0] = s_run_1524[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4819,14 +5098,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_333 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_365 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_334 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_366 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_24; } } float tot_1693 = s_run_1524[0]; @@ -4835,35 +5114,35 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; float send_13 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_367; float send_14 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_368; float send_15 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_369; float send_16 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_370; float v2_1702[2]; float send_17 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_371; float send_18 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_372; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_1693) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4892,9 +5171,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_16 = offset_1710 + _tmem_load_29[0]; if (t_49 < clen) { { - auto _gst_345 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_16; + auto _gst_377 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_16; } } int t_50 = 8 * th + 1; @@ -4902,9 +5181,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_17 = offset_1710 + _tmem_load_29[1]; if (t_50 < clen) { { - auto _gst_346 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_17; + auto _gst_378 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_17; } } int t_51 = 8 * th + 2; @@ -4912,9 +5191,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_18 = offset_1710 + _tmem_load_29[2]; if (t_51 < clen) { { - auto _gst_347 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_18; + auto _gst_379 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_18; } } int t_52 = 8 * th + 3; @@ -4922,9 +5201,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_19 = offset_1710 + _tmem_load_29[3]; if (t_52 < clen) { { - auto _gst_348 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_19; + auto _gst_380 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_19; } } int t_53 = 8 * th + 4; @@ -4932,9 +5211,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_20 = offset_1710 + _tmem_load_29[4]; if (t_53 < clen) { { - auto _gst_349 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_20; + auto _gst_381 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_20; } } int t_54 = 8 * th + 5; @@ -4942,9 +5221,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_21 = offset_1710 + _tmem_load_29[5]; if (t_54 < clen) { { - auto _gst_350 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_21; + auto _gst_382 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_21; } } int t_55 = 8 * th + 6; @@ -4952,9 +5231,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_22 = offset_1710 + _tmem_load_29[6]; if (t_55 < clen) { { - auto _gst_351 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_22; + auto _gst_383 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_22; } } int t_56 = 8 * th + 7; @@ -4962,9 +5241,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_23 = offset_1710 + _tmem_load_29[7]; if (t_56 < clen) { { - auto _gst_352 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_23; + auto _gst_384 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1708 + t_lo_1709; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_binding.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu similarity index 98% rename from flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_binding.cu rename to flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu index 680c8258f80..111665227c8 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_binding.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_kda_det_train_9ad80f98f22e0b3bce08, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_kda_det_train_ad4c93dbb0435933c8a1, for the device source that defines it. Do not edit. #include #include #include @@ -23,13 +23,13 @@ #include -extern "C" __global__ void kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride); +extern "C" __global__ void kernel_cake_kda_det_train_ad4c93dbb0435933c8a1(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride); -namespace cake_binding_kernel_cake_kda_det_train_9ad80f98f22e0b3bce08 { +namespace cake_binding_kernel_cake_kda_det_train_ad4c93dbb0435933c8a1 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_kda_det_train_9ad80f98f22e0b3bce08"; +constexpr const char* KERNEL_NAME = "kernel_cake_kda_det_train_ad4c93dbb0435933c8a1"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -487,7 +487,7 @@ void Run(TensorView a_dAqk, TensorView a_dAkk, TensorView a_gk, TensorView a_k_e const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { CheckCuda(cudaSetDevice(dev.device_id), "cudaSetDevice"); - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_kda_det_train_9ad80f98f22e0b3bce08), cudaFuncAttributeMaxDynamicSharedMemorySize, 115200), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_kda_det_train_ad4c93dbb0435933c8a1), cudaFuncAttributeMaxDynamicSharedMemorySize, 115200), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -527,11 +527,11 @@ void Run(TensorView a_dAqk, TensorView a_dAkk, TensorView a_gk, TensorView a_k_e config.blockDim = dim3(288, 1, 1); config.dynamicSmemBytes = 115200; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_kda_det_train_9ad80f98f22e0b3bce08), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_kda_det_train_ad4c93dbb0435933c8a1), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_kda_det_train_9ad80f98f22e0b3bce08 +} // namespace cake_binding_kernel_cake_kda_det_train_ad4c93dbb0435933c8a1 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_kda_det_train_9ad80f98f22e0b3bce08::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_kda_det_train_ad4c93dbb0435933c8a1::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu similarity index 85% rename from flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu rename to flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu index 63210c6849e..652b450a3bb 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -182,6 +189,21 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { } +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + __device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { float r; asm("mul.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); @@ -191,7 +213,7 @@ __device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void -kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride) +kernel_cake_kda_det_train_ad4c93dbb0435933c8a1(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -2256,12 +2278,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_3 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * e2_3); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_3, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_3 : __fmaf_rn(dl, _rcp_170, e2_3)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2285,12 +2313,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_4 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * e2_4); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_4, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_4, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_4 : __fmaf_rn(dl_1, _rcp_174, e2_4)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2314,12 +2348,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_5 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * e2_5); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_5, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_5, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_5 : __fmaf_rn(dl_3, _rcp_178, e2_5)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2343,12 +2383,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_6 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * e2_6); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_6, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_6, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_6 : __fmaf_rn(dl_5, _rcp_182, e2_6)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2384,12 +2430,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_2 = 48 + 8 * th + 2; @@ -2409,12 +2461,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_1 = approx_exp2(dc2_2.x); float e2_1_1 = approx_exp2(dc2_2.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_190 = approx_rcp(k_t0_2); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_191 = approx_rcp(k_t1_2); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_2.x; pan_641[3] = p2_2.y; int t0_3 = 48 + 8 * th + 4; @@ -2434,12 +2492,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_2 = approx_exp2(dc2_3.x); float e2_1_2 = approx_exp2(dc2_3.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_194 = approx_rcp(k_t0_3); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_195 = approx_rcp(k_t1_3); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_3.x; pan_641[5] = p2_3.y; int t0_4 = 48 + 8 * th + 6; @@ -2459,12 +2523,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(_f2_61, make_float2(g_t0_4, g_t1_4)); float e2_0_3 = approx_exp2(dc2_4.x); float e2_1_3 = approx_exp2(dc2_4.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_198 = approx_rcp(k_t0_4); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_199 = approx_rcp(k_t1_4); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_4.x; pan_641[7] = p2_4.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2512,12 +2582,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_202 = approx_rcp(k_t0_5); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_203 = approx_rcp(k_t1_5); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_5.x; pan_703[1] = p2_5.y; int t0_6 = 8 * th + 2; @@ -2541,12 +2617,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_2.x); float en_1_1 = approx_exp2(dn2_2.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_206 = approx_rcp(k_t0_6); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_207 = approx_rcp(k_t1_6); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_6.x; pan_703[3] = p2_6.y; int t0_7 = 8 * th + 4; @@ -2570,12 +2652,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_3.x); float en_1_2 = approx_exp2(dn2_3.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_210 = approx_rcp(k_t0_7); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_211 = approx_rcp(k_t1_7); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_7.x; pan_703[5] = p2_7.y; int t0_8 = 8 * th + 6; @@ -2599,12 +2687,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_4.x); float en_1_3 = approx_exp2(dn2_4.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_214 = approx_rcp(k_t0_8); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_215 = approx_rcp(k_t1_8); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_8.x; pan_703[7] = p2_8.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2646,12 +2740,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_5.x); float en_1_4 = approx_exp2(dn2_5.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_218 = approx_rcp(k_t0_9); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_219 = approx_rcp(k_t1_9); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_9.x; pan_777[1] = p2_9.y; int t0_10 = 16 + 8 * th + 2; @@ -2675,12 +2775,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_6.x); float en_1_5 = approx_exp2(dn2_6.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_222 = approx_rcp(k_t0_10); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_223 = approx_rcp(k_t1_10); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_10.x; pan_777[3] = p2_10.y; int t0_11 = 16 + 8 * th + 4; @@ -2704,12 +2810,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_7.x); float en_1_6 = approx_exp2(dn2_7.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_226 = approx_rcp(k_t0_11); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_227 = approx_rcp(k_t1_11); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_11.x; pan_777[5] = p2_11.y; int t0_12 = 16 + 8 * th + 6; @@ -2733,12 +2845,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_8 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_8.x); float en_1_7 = approx_exp2(dn2_8.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_230 = approx_rcp(k_t0_12); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_231 = approx_rcp(k_t1_12); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_12.x; pan_777[7] = p2_12.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2809,37 +2927,37 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_4 = approx_exp2(d2_57.x); float e1_4 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_4, e1_4); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_4, e1_4); float2 v2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_4.x; v_562_0[1] = v2_4.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_5 = approx_exp2(d2_58.x); float e1_5 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_5, e1_5); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_5, e1_5); float2 v2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_5.x; v_562_0[3] = v2_5.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_6 = approx_exp2(d2_59.x); float e1_6 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_6, e1_6); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_6, e1_6); float2 v2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_6.x; v_562_0[5] = v2_6.y; float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_7 = approx_exp2(d2_60.x); float e1_7 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_7, e1_7); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_7, e1_7); float2 v2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_7.x; v_562_0[7] = v2_7.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2851,29 +2969,29 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2881,29 +2999,29 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2973,7 +3091,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3005,7 +3128,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3037,7 +3165,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_2)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3069,7 +3202,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_3)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3101,7 +3239,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_4)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3133,7 +3276,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_5)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3165,7 +3313,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_6)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3197,7 +3350,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_7)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3212,41 +3370,41 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_216 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_216; + float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_248; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_217 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_217; + float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_249; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_218 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_218; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_250; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_219 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_219; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_251; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_220 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_220; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_252; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_221 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_221; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_253; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_222 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_222; - float _shfl_xor_sync_223 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_223; - float _shfl_xor_sync_224 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_224; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_254; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_255; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_256; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); int ew_0 = 98304 - 32768 * th >> 2; float nz = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); bool live0 = 32 + 8 * th + qd < clen; bool live1 = 32 + 8 * th + qd + 4 < clen; - int _min_225 = ((32 + 8 * th + qd) < (clen - 1) ? (32 + 8 * th + qd) : (clen - 1)); - long long rowa = ext0 + (long long)_min_225; - int _min_226 = ((32 + 8 * th + qd + 4) < (clen - 1) ? (32 + 8 * th + qd + 4) : (clen - 1)); - long long rowb = ext0 + (long long)_min_226; + int _min_257 = ((32 + 8 * th + qd) < (clen - 1) ? (32 + 8 * th + qd) : (clen - 1)); + long long rowa = ext0 + (long long)_min_257; + int _min_258 = ((32 + 8 * th + qd + 4) < (clen - 1) ? (32 + 8 * th + qd + 4) : (clen - 1)); + long long rowb = ext0 + (long long)_min_258; float xqa[4]; float xka[4]; float xqb[4]; @@ -3274,52 +3432,52 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (ew_0 + 1024 + 128 * (qd + 4) + 4 * lane_1)))); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint2 _vld_227; - _vld_227 = *reinterpret_cast(q_raw + (rowa * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_227 = reinterpret_cast(&_vld_227); + uint2 _vld_259; + _vld_259 = *reinterpret_cast(q_raw + (rowa * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_259 = reinterpret_cast(&_vld_259); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqa[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_227[_pair]) << 16); - (&xqa[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_227[_pair]) & 0xffff0000u); + (&xqa[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_259[_pair]) << 16); + (&xqa[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_259[_pair]) & 0xffff0000u); } } { rqa[0] = *reinterpret_cast(qk_rstd + ((0 + rowa) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_228; - _vld_228 = *reinterpret_cast(k_raw + (rowa * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_228 = reinterpret_cast(&_vld_228); + uint2 _vld_260; + _vld_260 = *reinterpret_cast(k_raw + (rowa * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_260 = reinterpret_cast(&_vld_260); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xka[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_228[_pair]) << 16); - (&xka[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_228[_pair]) & 0xffff0000u); + (&xka[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_260[_pair]) << 16); + (&xka[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_260[_pair]) & 0xffff0000u); } } { rka[0] = *reinterpret_cast(qk_rstd + (((long long)num_tokens + rowa) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_229; - _vld_229 = *reinterpret_cast(q_raw + (rowb * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_229 = reinterpret_cast(&_vld_229); + uint2 _vld_261; + _vld_261 = *reinterpret_cast(q_raw + (rowb * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_261 = reinterpret_cast(&_vld_261); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqb[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_229[_pair]) << 16); - (&xqb[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_229[_pair]) & 0xffff0000u); + (&xqb[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_261[_pair]) << 16); + (&xqb[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_261[_pair]) & 0xffff0000u); } } { rqb[0] = *reinterpret_cast(qk_rstd + ((0 + rowb) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_230; - _vld_230 = *reinterpret_cast(k_raw + (rowb * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_230 = reinterpret_cast(&_vld_230); + uint2 _vld_262; + _vld_262 = *reinterpret_cast(k_raw + (rowb * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_262 = reinterpret_cast(&_vld_262); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xkb[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_230[_pair]) << 16); - (&xkb[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_230[_pair]) & 0xffff0000u); + (&xkb[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_262[_pair]) << 16); + (&xkb[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_262[_pair]) & 0xffff0000u); } } { @@ -3359,106 +3517,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y[4]; float p[4]; y[0] = mul_f32_rn_noftz(xq[0], rq[0]); - float _fma_231; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_231) : "f"(gq[0]), "f"(y[0]), "f"(nz)); - p[0] = _fma_231; + float _fma_263; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_263) : "f"(gq[0]), "f"(y[0]), "f"(nz)); + p[0] = _fma_263; y[1] = mul_f32_rn_noftz(xq[1], rq[0]); - float _fma_232; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_232) : "f"(gq[1]), "f"(y[1]), "f"(nz)); - p[1] = _fma_232; + float _fma_264; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_264) : "f"(gq[1]), "f"(y[1]), "f"(nz)); + p[1] = _fma_264; y[2] = mul_f32_rn_noftz(xq[2], rq[0]); - float _fma_233; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_233) : "f"(gq[2]), "f"(y[2]), "f"(nz)); - p[2] = _fma_233; + float _fma_265; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_265) : "f"(gq[2]), "f"(y[2]), "f"(nz)); + p[2] = _fma_265; y[3] = mul_f32_rn_noftz(xq[3], rq[0]); - float _fma_234; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_234) : "f"(gq[3]), "f"(y[3]), "f"(nz)); - p[3] = _fma_234; - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, p[0], 4); - float _add_235; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_235) : "f"(p[0]), "f"(_shfl_xor_sync_236)); - p[0] = _add_235; - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, p[1], 4); - float _add_237; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_237) : "f"(p[1]), "f"(_shfl_xor_sync_238)); - p[1] = _add_237; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, p[2], 4); - float _add_239; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_239) : "f"(p[2]), "f"(_shfl_xor_sync_240)); - p[2] = _add_239; - float _shfl_xor_sync_242 = __shfl_xor_sync(0xFFFFFFFF, p[3], 4); - float _add_241; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_241) : "f"(p[3]), "f"(_shfl_xor_sync_242)); - p[3] = _add_241; - float _shfl_xor_sync_244 = __shfl_xor_sync(0xFFFFFFFF, p[0], 2); - float _add_243; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_243) : "f"(p[0]), "f"(_shfl_xor_sync_244)); - p[0] = _add_243; - float _shfl_xor_sync_246 = __shfl_xor_sync(0xFFFFFFFF, p[1], 2); - float _add_245; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_245) : "f"(p[1]), "f"(_shfl_xor_sync_246)); - p[1] = _add_245; - float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, p[2], 2); - float _add_247; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_247) : "f"(p[2]), "f"(_shfl_xor_sync_248)); - p[2] = _add_247; - float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, p[3], 2); - float _add_249; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_249) : "f"(p[3]), "f"(_shfl_xor_sync_250)); - p[3] = _add_249; - float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, p[0], 1); - float _add_251; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_251) : "f"(p[0]), "f"(_shfl_xor_sync_252)); - p[0] = _add_251; - float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, p[1], 1); - float _add_253; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_253) : "f"(p[1]), "f"(_shfl_xor_sync_254)); - p[1] = _add_253; - float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, p[2], 1); - float _add_255; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_255) : "f"(p[2]), "f"(_shfl_xor_sync_256)); - p[2] = _add_255; - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, p[3], 1); - float _add_257; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_257) : "f"(p[3]), "f"(_shfl_xor_sync_258)); - p[3] = _add_257; - float _add_259; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_259) : "f"(p[0]), "f"(p[2])); - float _add_260; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_260) : "f"(p[1]), "f"(p[3])); + float _fma_266; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_266) : "f"(gq[3]), "f"(y[3]), "f"(nz)); + p[3] = _fma_266; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, p[0], 4); + float _add_267; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_267) : "f"(p[0]), "f"(_shfl_xor_sync_268)); + p[0] = _add_267; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, p[1], 4); + float _add_269; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_269) : "f"(p[1]), "f"(_shfl_xor_sync_270)); + p[1] = _add_269; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, p[2], 4); + float _add_271; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_271) : "f"(p[2]), "f"(_shfl_xor_sync_272)); + p[2] = _add_271; + float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, p[3], 4); + float _add_273; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_273) : "f"(p[3]), "f"(_shfl_xor_sync_274)); + p[3] = _add_273; + float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, p[0], 2); + float _add_275; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_275) : "f"(p[0]), "f"(_shfl_xor_sync_276)); + p[0] = _add_275; + float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, p[1], 2); + float _add_277; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_277) : "f"(p[1]), "f"(_shfl_xor_sync_278)); + p[1] = _add_277; + float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, p[2], 2); + float _add_279; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_279) : "f"(p[2]), "f"(_shfl_xor_sync_280)); + p[2] = _add_279; + float _shfl_xor_sync_282 = __shfl_xor_sync(0xFFFFFFFF, p[3], 2); + float _add_281; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_281) : "f"(p[3]), "f"(_shfl_xor_sync_282)); + p[3] = _add_281; + float _shfl_xor_sync_284 = __shfl_xor_sync(0xFFFFFFFF, p[0], 1); + float _add_283; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_283) : "f"(p[0]), "f"(_shfl_xor_sync_284)); + p[0] = _add_283; + float _shfl_xor_sync_286 = __shfl_xor_sync(0xFFFFFFFF, p[1], 1); + float _add_285; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_285) : "f"(p[1]), "f"(_shfl_xor_sync_286)); + p[1] = _add_285; + float _shfl_xor_sync_288 = __shfl_xor_sync(0xFFFFFFFF, p[2], 1); + float _add_287; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_287) : "f"(p[2]), "f"(_shfl_xor_sync_288)); + p[2] = _add_287; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, p[3], 1); + float _add_289; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_289) : "f"(p[3]), "f"(_shfl_xor_sync_290)); + p[3] = _add_289; + float _add_291; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_291) : "f"(p[0]), "f"(p[2])); + float _add_292; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_292) : "f"(p[1]), "f"(p[3])); float sq; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq) : "f"(_add_259), "f"(_add_260)); - float _shfl_sync_263 = __shfl_sync(0xFFFFFFFF, sq, 0); - float _shfl_sync_264 = __shfl_sync(0xFFFFFFFF, sq, 8); - float _add_262; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_262) : "f"(_shfl_sync_263), "f"(_shfl_sync_264)); - float _shfl_sync_265 = __shfl_sync(0xFFFFFFFF, sq, 16); - float _add_261; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_261) : "f"(_add_262), "f"(_shfl_sync_265)); - float _shfl_sync_266 = __shfl_sync(0xFFFFFFFF, sq, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq) : "f"(_add_291), "f"(_add_292)); + float _shfl_sync_295 = __shfl_sync(0xFFFFFFFF, sq, 0); + float _shfl_sync_296 = __shfl_sync(0xFFFFFFFF, sq, 8); + float _add_294; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_294) : "f"(_shfl_sync_295), "f"(_shfl_sync_296)); + float _shfl_sync_297 = __shfl_sync(0xFFFFFFFF, sq, 16); + float _add_293; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_293) : "f"(_add_294), "f"(_shfl_sync_297)); + float _shfl_sync_298 = __shfl_sync(0xFFFFFFFF, sq, 24); float dot; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_261), "f"(_shfl_sync_266)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_293), "f"(_shfl_sync_298)); float o[4]; - float _fma_268; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_268) : "f"(y[0]), "f"(dot), "f"(nz)); - float _sub_267; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_267) : "f"(gq[0]), "f"(_fma_268)); - o[0] = mul_f32_rn_noftz(rq[0], _sub_267); - float _fma_270; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_270) : "f"(y[1]), "f"(dot), "f"(nz)); - float _sub_269; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_269) : "f"(gq[1]), "f"(_fma_270)); - o[1] = mul_f32_rn_noftz(rq[0], _sub_269); - float _fma_272; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_272) : "f"(y[2]), "f"(dot), "f"(nz)); - float _sub_271; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_271) : "f"(gq[2]), "f"(_fma_272)); - o[2] = mul_f32_rn_noftz(rq[0], _sub_271); - float _fma_274; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_274) : "f"(y[3]), "f"(dot), "f"(nz)); - float _sub_273; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_273) : "f"(gq[3]), "f"(_fma_274)); - o[3] = mul_f32_rn_noftz(rq[0], _sub_273); + float _fma_300; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_300) : "f"(y[0]), "f"(dot), "f"(nz)); + float _sub_299; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_299) : "f"(gq[0]), "f"(_fma_300)); + o[0] = mul_f32_rn_noftz(rq[0], _sub_299); + float _fma_302; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_302) : "f"(y[1]), "f"(dot), "f"(nz)); + float _sub_301; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_301) : "f"(gq[1]), "f"(_fma_302)); + o[1] = mul_f32_rn_noftz(rq[0], _sub_301); + float _fma_304; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_304) : "f"(y[2]), "f"(dot), "f"(nz)); + float _sub_303; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_303) : "f"(gq[2]), "f"(_fma_304)); + o[2] = mul_f32_rn_noftz(rq[0], _sub_303); + float _fma_306; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_306) : "f"(y[3]), "f"(dot), "f"(nz)); + float _sub_305; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_305) : "f"(gq[3]), "f"(_fma_306)); + o[3] = mul_f32_rn_noftz(rq[0], _sub_305); unsigned int ob[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -3469,106 +3627,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_0[4]; float p_0[4]; y_0[0] = mul_f32_rn_noftz(xk[0], rk[0]); - float _fma_275; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_275) : "f"(gk_0[0]), "f"(y_0[0]), "f"(nz)); - p_0[0] = _fma_275; + float _fma_307; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_307) : "f"(gk_0[0]), "f"(y_0[0]), "f"(nz)); + p_0[0] = _fma_307; y_0[1] = mul_f32_rn_noftz(xk[1], rk[0]); - float _fma_276; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_276) : "f"(gk_0[1]), "f"(y_0[1]), "f"(nz)); - p_0[1] = _fma_276; + float _fma_308; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_308) : "f"(gk_0[1]), "f"(y_0[1]), "f"(nz)); + p_0[1] = _fma_308; y_0[2] = mul_f32_rn_noftz(xk[2], rk[0]); - float _fma_277; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_277) : "f"(gk_0[2]), "f"(y_0[2]), "f"(nz)); - p_0[2] = _fma_277; + float _fma_309; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_309) : "f"(gk_0[2]), "f"(y_0[2]), "f"(nz)); + p_0[2] = _fma_309; y_0[3] = mul_f32_rn_noftz(xk[3], rk[0]); - float _fma_278; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_278) : "f"(gk_0[3]), "f"(y_0[3]), "f"(nz)); - p_0[3] = _fma_278; - float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 4); - float _add_279; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_279) : "f"(p_0[0]), "f"(_shfl_xor_sync_280)); - p_0[0] = _add_279; - float _shfl_xor_sync_282 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 4); - float _add_281; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_281) : "f"(p_0[1]), "f"(_shfl_xor_sync_282)); - p_0[1] = _add_281; - float _shfl_xor_sync_284 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 4); - float _add_283; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_283) : "f"(p_0[2]), "f"(_shfl_xor_sync_284)); - p_0[2] = _add_283; - float _shfl_xor_sync_286 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 4); - float _add_285; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_285) : "f"(p_0[3]), "f"(_shfl_xor_sync_286)); - p_0[3] = _add_285; - float _shfl_xor_sync_288 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 2); - float _add_287; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_287) : "f"(p_0[0]), "f"(_shfl_xor_sync_288)); - p_0[0] = _add_287; - float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 2); - float _add_289; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_289) : "f"(p_0[1]), "f"(_shfl_xor_sync_290)); - p_0[1] = _add_289; - float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 2); - float _add_291; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_291) : "f"(p_0[2]), "f"(_shfl_xor_sync_292)); - p_0[2] = _add_291; - float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 2); - float _add_293; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_293) : "f"(p_0[3]), "f"(_shfl_xor_sync_294)); - p_0[3] = _add_293; - float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 1); - float _add_295; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_295) : "f"(p_0[0]), "f"(_shfl_xor_sync_296)); - p_0[0] = _add_295; - float _shfl_xor_sync_298 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 1); - float _add_297; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_297) : "f"(p_0[1]), "f"(_shfl_xor_sync_298)); - p_0[1] = _add_297; - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 1); - float _add_299; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_299) : "f"(p_0[2]), "f"(_shfl_xor_sync_300)); - p_0[2] = _add_299; - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 1); - float _add_301; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_301) : "f"(p_0[3]), "f"(_shfl_xor_sync_302)); - p_0[3] = _add_301; - float _add_303; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_303) : "f"(p_0[0]), "f"(p_0[2])); - float _add_304; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_304) : "f"(p_0[1]), "f"(p_0[3])); + float _fma_310; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_310) : "f"(gk_0[3]), "f"(y_0[3]), "f"(nz)); + p_0[3] = _fma_310; + float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 4); + float _add_311; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_311) : "f"(p_0[0]), "f"(_shfl_xor_sync_312)); + p_0[0] = _add_311; + float _shfl_xor_sync_314 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 4); + float _add_313; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_313) : "f"(p_0[1]), "f"(_shfl_xor_sync_314)); + p_0[1] = _add_313; + float _shfl_xor_sync_316 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 4); + float _add_315; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_315) : "f"(p_0[2]), "f"(_shfl_xor_sync_316)); + p_0[2] = _add_315; + float _shfl_xor_sync_318 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 4); + float _add_317; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_317) : "f"(p_0[3]), "f"(_shfl_xor_sync_318)); + p_0[3] = _add_317; + float _shfl_xor_sync_320 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 2); + float _add_319; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_319) : "f"(p_0[0]), "f"(_shfl_xor_sync_320)); + p_0[0] = _add_319; + float _shfl_xor_sync_322 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 2); + float _add_321; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_321) : "f"(p_0[1]), "f"(_shfl_xor_sync_322)); + p_0[1] = _add_321; + float _shfl_xor_sync_324 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 2); + float _add_323; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_323) : "f"(p_0[2]), "f"(_shfl_xor_sync_324)); + p_0[2] = _add_323; + float _shfl_xor_sync_326 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 2); + float _add_325; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_325) : "f"(p_0[3]), "f"(_shfl_xor_sync_326)); + p_0[3] = _add_325; + float _shfl_xor_sync_328 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 1); + float _add_327; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_327) : "f"(p_0[0]), "f"(_shfl_xor_sync_328)); + p_0[0] = _add_327; + float _shfl_xor_sync_330 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 1); + float _add_329; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_329) : "f"(p_0[1]), "f"(_shfl_xor_sync_330)); + p_0[1] = _add_329; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 1); + float _add_331; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_331) : "f"(p_0[2]), "f"(_shfl_xor_sync_332)); + p_0[2] = _add_331; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 1); + float _add_333; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_333) : "f"(p_0[3]), "f"(_shfl_xor_sync_334)); + p_0[3] = _add_333; + float _add_335; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_335) : "f"(p_0[0]), "f"(p_0[2])); + float _add_336; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_336) : "f"(p_0[1]), "f"(p_0[3])); float sq_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_0) : "f"(_add_303), "f"(_add_304)); - float _shfl_sync_307 = __shfl_sync(0xFFFFFFFF, sq_0, 0); - float _shfl_sync_308 = __shfl_sync(0xFFFFFFFF, sq_0, 8); - float _add_306; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_306) : "f"(_shfl_sync_307), "f"(_shfl_sync_308)); - float _shfl_sync_309 = __shfl_sync(0xFFFFFFFF, sq_0, 16); - float _add_305; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_305) : "f"(_add_306), "f"(_shfl_sync_309)); - float _shfl_sync_310 = __shfl_sync(0xFFFFFFFF, sq_0, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_0) : "f"(_add_335), "f"(_add_336)); + float _shfl_sync_339 = __shfl_sync(0xFFFFFFFF, sq_0, 0); + float _shfl_sync_340 = __shfl_sync(0xFFFFFFFF, sq_0, 8); + float _add_338; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_338) : "f"(_shfl_sync_339), "f"(_shfl_sync_340)); + float _shfl_sync_341 = __shfl_sync(0xFFFFFFFF, sq_0, 16); + float _add_337; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_337) : "f"(_add_338), "f"(_shfl_sync_341)); + float _shfl_sync_342 = __shfl_sync(0xFFFFFFFF, sq_0, 24); float dot_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_305), "f"(_shfl_sync_310)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_337), "f"(_shfl_sync_342)); float o_0[4]; - float _fma_312; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_312) : "f"(y_0[0]), "f"(dot_0), "f"(nz)); - float _sub_311; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_311) : "f"(gk_0[0]), "f"(_fma_312)); - o_0[0] = mul_f32_rn_noftz(rk[0], _sub_311); - float _fma_314; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_314) : "f"(y_0[1]), "f"(dot_0), "f"(nz)); - float _sub_313; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_313) : "f"(gk_0[1]), "f"(_fma_314)); - o_0[1] = mul_f32_rn_noftz(rk[0], _sub_313); - float _fma_316; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_316) : "f"(y_0[2]), "f"(dot_0), "f"(nz)); - float _sub_315; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_315) : "f"(gk_0[2]), "f"(_fma_316)); - o_0[2] = mul_f32_rn_noftz(rk[0], _sub_315); - float _fma_318; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_318) : "f"(y_0[3]), "f"(dot_0), "f"(nz)); - float _sub_317; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_317) : "f"(gk_0[3]), "f"(_fma_318)); - o_0[3] = mul_f32_rn_noftz(rk[0], _sub_317); + float _fma_344; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_344) : "f"(y_0[0]), "f"(dot_0), "f"(nz)); + float _sub_343; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_343) : "f"(gk_0[0]), "f"(_fma_344)); + o_0[0] = mul_f32_rn_noftz(rk[0], _sub_343); + float _fma_346; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_346) : "f"(y_0[1]), "f"(dot_0), "f"(nz)); + float _sub_345; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_345) : "f"(gk_0[1]), "f"(_fma_346)); + o_0[1] = mul_f32_rn_noftz(rk[0], _sub_345); + float _fma_348; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_348) : "f"(y_0[2]), "f"(dot_0), "f"(nz)); + float _sub_347; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_347) : "f"(gk_0[2]), "f"(_fma_348)); + o_0[2] = mul_f32_rn_noftz(rk[0], _sub_347); + float _fma_350; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_350) : "f"(y_0[3]), "f"(dot_0), "f"(nz)); + float _sub_349; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_349) : "f"(gk_0[3]), "f"(_fma_350)); + o_0[3] = mul_f32_rn_noftz(rk[0], _sub_349); unsigned int ob_0[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -3633,7 +3791,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; + float qe_7 = q_t_1_8 * e_dg_0; + float kp_19 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_0; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_14[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_0) * _tmem_load_15[7]); + float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_16[7], k_t_0, cr_8)) + dgf_0; s_run_1059[0] = s_run_1059[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3665,7 +3828,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[6]; - float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, (dk2b_2 - _tmem_load_16[6]) * k_t_2) + dgf_2; + float qe_8 = q_t_1_9 * e_dg_2; + float kp_20 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_14[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_2) * _tmem_load_15[6]); + float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_16[6], k_t_2, cr_9)) + dgf_2; s_run_1059[0] = s_run_1059[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3697,7 +3865,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[5]; - float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, (dk2b_3 - _tmem_load_16[5]) * k_t_3) + dgf_3; + float qe_9 = q_t_1_10 * e_dg_3; + float kp_21 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_3; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_14[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_3) * _tmem_load_15[5]); + float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_16[5], k_t_3, cr_10)) + dgf_3; s_run_1059[0] = s_run_1059[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3729,7 +3902,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[4]; - float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, (dk2b_4 - _tmem_load_16[4]) * k_t_4) + dgf_4; + float qe_10 = q_t_1_11 * e_dg_4; + float kp_22 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_4; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_14[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_4) * _tmem_load_15[4]); + float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_16[4], k_t_4, cr_11)) + dgf_4; s_run_1059[0] = s_run_1059[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3761,7 +3939,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[3]; - float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, (dk2b_5 - _tmem_load_16[3]) * k_t_5) + dgf_5; + float qe_11 = q_t_1_12 * e_dg_5; + float kp_23 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_5; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_14[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_5) * _tmem_load_15[3]); + float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_16[3], k_t_5, cr_12)) + dgf_5; s_run_1059[0] = s_run_1059[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3793,7 +3976,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[2]; - float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, (dk2b_6 - _tmem_load_16[2]) * k_t_6) + dgf_6; + float qe_12 = q_t_1_13 * e_dg_6; + float kp_24 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_6; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_14[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_6) * _tmem_load_15[2]); + float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_16[2], k_t_6, cr_13)) + dgf_6; s_run_1059[0] = s_run_1059[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3825,7 +4013,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[1]; - float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, (dk2b_7 - _tmem_load_16[1]) * k_t_7) + dgf_7; + float qe_13 = q_t_1_14 * e_dg_7; + float kp_25 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_7; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_14[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_7) * _tmem_load_15[1]); + float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_16[1], k_t_7, cr_14)) + dgf_7; s_run_1059[0] = s_run_1059[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3857,7 +4050,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_16[0]; - float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, (dk2b_8 - _tmem_load_16[0]) * k_t_8) + dgf_8; + float qe_14 = q_t_1_15 * e_dg_8; + float kp_26 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_8; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_14[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_8) * _tmem_load_15[0]); + float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_16[0], k_t_8, cr_15)) + dgf_8; s_run_1059[0] = s_run_1059[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3872,40 +4070,40 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int b2_1231 = lane_1 >> 2 & 1; float v4_1232[4]; float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_319 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_319; + float _shfl_xor_sync_351 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_351; float send_2 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_320 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_320; + float _shfl_xor_sync_352 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_352; float send_3 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_321 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_321; + float _shfl_xor_sync_353 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_353; float send_4 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_322 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_322; + float _shfl_xor_sync_354 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_354; float v2_1237[2]; float send_5 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_323 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_323; + float _shfl_xor_sync_355 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_355; float send_6 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_324 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_324; + float _shfl_xor_sync_356 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); + v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_356; float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_325 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_325; - float _shfl_xor_sync_326 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_326; - float _shfl_xor_sync_327 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_327; + float _shfl_xor_sync_357 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); + float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_357; + float _shfl_xor_sync_358 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); + v1_1241 = v1_1241 + _shfl_xor_sync_358; + float _shfl_xor_sync_359 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); + v1_1241 = v1_1241 + _shfl_xor_sync_359; asm volatile("tcgen05.fence::before_thread_sync;"); int ew_1 = 98304 - 32768 * th >> 2; float nz_0 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); bool live0_0 = 48 + 8 * th + qd < clen; bool live1_0 = 48 + 8 * th + qd + 4 < clen; - int _min_328 = ((48 + 8 * th + qd) < (clen - 1) ? (48 + 8 * th + qd) : (clen - 1)); - long long rowa_0 = ext0 + (long long)_min_328; - int _min_329 = ((48 + 8 * th + qd + 4) < (clen - 1) ? (48 + 8 * th + qd + 4) : (clen - 1)); - long long rowb_0 = ext0 + (long long)_min_329; + int _min_360 = ((48 + 8 * th + qd) < (clen - 1) ? (48 + 8 * th + qd) : (clen - 1)); + long long rowa_0 = ext0 + (long long)_min_360; + int _min_361 = ((48 + 8 * th + qd + 4) < (clen - 1) ? (48 + 8 * th + qd + 4) : (clen - 1)); + long long rowb_0 = ext0 + (long long)_min_361; float xqa_0[4]; float xka_0[4]; float xqb_0[4]; @@ -3933,52 +4131,52 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (ew_1 + 1024 + 128 * (qd + 4) + 4 * lane_1)))); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint2 _vld_330; - _vld_330 = *reinterpret_cast(q_raw + (rowa_0 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_330 = reinterpret_cast(&_vld_330); + uint2 _vld_362; + _vld_362 = *reinterpret_cast(q_raw + (rowa_0 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_362 = reinterpret_cast(&_vld_362); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqa_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_330[_pair]) << 16); - (&xqa_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_330[_pair]) & 0xffff0000u); + (&xqa_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_362[_pair]) << 16); + (&xqa_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_362[_pair]) & 0xffff0000u); } } { rqa_0[0] = *reinterpret_cast(qk_rstd + ((0 + rowa_0) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_331; - _vld_331 = *reinterpret_cast(k_raw + (rowa_0 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_331 = reinterpret_cast(&_vld_331); + uint2 _vld_363; + _vld_363 = *reinterpret_cast(k_raw + (rowa_0 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_363 = reinterpret_cast(&_vld_363); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xka_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_331[_pair]) << 16); - (&xka_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_331[_pair]) & 0xffff0000u); + (&xka_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_363[_pair]) << 16); + (&xka_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_363[_pair]) & 0xffff0000u); } } { rka_0[0] = *reinterpret_cast(qk_rstd + (((long long)num_tokens + rowa_0) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_332; - _vld_332 = *reinterpret_cast(q_raw + (rowb_0 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_332 = reinterpret_cast(&_vld_332); + uint2 _vld_364; + _vld_364 = *reinterpret_cast(q_raw + (rowb_0 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_364 = reinterpret_cast(&_vld_364); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqb_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_332[_pair]) << 16); - (&xqb_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_332[_pair]) & 0xffff0000u); + (&xqb_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_364[_pair]) << 16); + (&xqb_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_364[_pair]) & 0xffff0000u); } } { rqb_0[0] = *reinterpret_cast(qk_rstd + ((0 + rowb_0) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_333; - _vld_333 = *reinterpret_cast(k_raw + (rowb_0 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_333 = reinterpret_cast(&_vld_333); + uint2 _vld_365; + _vld_365 = *reinterpret_cast(k_raw + (rowb_0 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_365 = reinterpret_cast(&_vld_365); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xkb_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_333[_pair]) << 16); - (&xkb_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_333[_pair]) & 0xffff0000u); + (&xkb_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_365[_pair]) << 16); + (&xkb_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_365[_pair]) & 0xffff0000u); } } { @@ -4018,106 +4216,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_1[4]; float p_1[4]; y_1[0] = mul_f32_rn_noftz(xq_1[0], rq_1[0]); - float _fma_334; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_334) : "f"(gq_1[0]), "f"(y_1[0]), "f"(nz_0)); - p_1[0] = _fma_334; + float _fma_366; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_366) : "f"(gq_1[0]), "f"(y_1[0]), "f"(nz_0)); + p_1[0] = _fma_366; y_1[1] = mul_f32_rn_noftz(xq_1[1], rq_1[0]); - float _fma_335; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_335) : "f"(gq_1[1]), "f"(y_1[1]), "f"(nz_0)); - p_1[1] = _fma_335; + float _fma_367; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_367) : "f"(gq_1[1]), "f"(y_1[1]), "f"(nz_0)); + p_1[1] = _fma_367; y_1[2] = mul_f32_rn_noftz(xq_1[2], rq_1[0]); - float _fma_336; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_336) : "f"(gq_1[2]), "f"(y_1[2]), "f"(nz_0)); - p_1[2] = _fma_336; + float _fma_368; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_368) : "f"(gq_1[2]), "f"(y_1[2]), "f"(nz_0)); + p_1[2] = _fma_368; y_1[3] = mul_f32_rn_noftz(xq_1[3], rq_1[0]); - float _fma_337; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_337) : "f"(gq_1[3]), "f"(y_1[3]), "f"(nz_0)); - p_1[3] = _fma_337; - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 4); - float _add_338; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_338) : "f"(p_1[0]), "f"(_shfl_xor_sync_339)); - p_1[0] = _add_338; - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 4); - float _add_340; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_340) : "f"(p_1[1]), "f"(_shfl_xor_sync_341)); - p_1[1] = _add_340; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 4); - float _add_342; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_342) : "f"(p_1[2]), "f"(_shfl_xor_sync_343)); - p_1[2] = _add_342; - float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 4); - float _add_344; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_344) : "f"(p_1[3]), "f"(_shfl_xor_sync_345)); - p_1[3] = _add_344; - float _shfl_xor_sync_347 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 2); - float _add_346; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_346) : "f"(p_1[0]), "f"(_shfl_xor_sync_347)); - p_1[0] = _add_346; - float _shfl_xor_sync_349 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 2); - float _add_348; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_348) : "f"(p_1[1]), "f"(_shfl_xor_sync_349)); - p_1[1] = _add_348; - float _shfl_xor_sync_351 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 2); - float _add_350; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_350) : "f"(p_1[2]), "f"(_shfl_xor_sync_351)); - p_1[2] = _add_350; - float _shfl_xor_sync_353 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 2); - float _add_352; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_352) : "f"(p_1[3]), "f"(_shfl_xor_sync_353)); - p_1[3] = _add_352; - float _shfl_xor_sync_355 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 1); - float _add_354; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_354) : "f"(p_1[0]), "f"(_shfl_xor_sync_355)); - p_1[0] = _add_354; - float _shfl_xor_sync_357 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 1); - float _add_356; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_356) : "f"(p_1[1]), "f"(_shfl_xor_sync_357)); - p_1[1] = _add_356; - float _shfl_xor_sync_359 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 1); - float _add_358; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_358) : "f"(p_1[2]), "f"(_shfl_xor_sync_359)); - p_1[2] = _add_358; - float _shfl_xor_sync_361 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 1); - float _add_360; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_360) : "f"(p_1[3]), "f"(_shfl_xor_sync_361)); - p_1[3] = _add_360; - float _add_362; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_362) : "f"(p_1[0]), "f"(p_1[2])); - float _add_363; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_363) : "f"(p_1[1]), "f"(p_1[3])); + float _fma_369; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_369) : "f"(gq_1[3]), "f"(y_1[3]), "f"(nz_0)); + p_1[3] = _fma_369; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 4); + float _add_370; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_370) : "f"(p_1[0]), "f"(_shfl_xor_sync_371)); + p_1[0] = _add_370; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 4); + float _add_372; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_372) : "f"(p_1[1]), "f"(_shfl_xor_sync_373)); + p_1[1] = _add_372; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 4); + float _add_374; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_374) : "f"(p_1[2]), "f"(_shfl_xor_sync_375)); + p_1[2] = _add_374; + float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 4); + float _add_376; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_376) : "f"(p_1[3]), "f"(_shfl_xor_sync_377)); + p_1[3] = _add_376; + float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 2); + float _add_378; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_378) : "f"(p_1[0]), "f"(_shfl_xor_sync_379)); + p_1[0] = _add_378; + float _shfl_xor_sync_381 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 2); + float _add_380; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_380) : "f"(p_1[1]), "f"(_shfl_xor_sync_381)); + p_1[1] = _add_380; + float _shfl_xor_sync_383 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 2); + float _add_382; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_382) : "f"(p_1[2]), "f"(_shfl_xor_sync_383)); + p_1[2] = _add_382; + float _shfl_xor_sync_385 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 2); + float _add_384; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_384) : "f"(p_1[3]), "f"(_shfl_xor_sync_385)); + p_1[3] = _add_384; + float _shfl_xor_sync_387 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 1); + float _add_386; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_386) : "f"(p_1[0]), "f"(_shfl_xor_sync_387)); + p_1[0] = _add_386; + float _shfl_xor_sync_389 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 1); + float _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(p_1[1]), "f"(_shfl_xor_sync_389)); + p_1[1] = _add_388; + float _shfl_xor_sync_391 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 1); + float _add_390; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_390) : "f"(p_1[2]), "f"(_shfl_xor_sync_391)); + p_1[2] = _add_390; + float _shfl_xor_sync_393 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 1); + float _add_392; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(p_1[3]), "f"(_shfl_xor_sync_393)); + p_1[3] = _add_392; + float _add_394; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(p_1[0]), "f"(p_1[2])); + float _add_395; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_395) : "f"(p_1[1]), "f"(p_1[3])); float sq_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1) : "f"(_add_362), "f"(_add_363)); - float _shfl_sync_366 = __shfl_sync(0xFFFFFFFF, sq_1, 0); - float _shfl_sync_367 = __shfl_sync(0xFFFFFFFF, sq_1, 8); - float _add_365; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_365) : "f"(_shfl_sync_366), "f"(_shfl_sync_367)); - float _shfl_sync_368 = __shfl_sync(0xFFFFFFFF, sq_1, 16); - float _add_364; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_364) : "f"(_add_365), "f"(_shfl_sync_368)); - float _shfl_sync_369 = __shfl_sync(0xFFFFFFFF, sq_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1) : "f"(_add_394), "f"(_add_395)); + float _shfl_sync_398 = __shfl_sync(0xFFFFFFFF, sq_1, 0); + float _shfl_sync_399 = __shfl_sync(0xFFFFFFFF, sq_1, 8); + float _add_397; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_397) : "f"(_shfl_sync_398), "f"(_shfl_sync_399)); + float _shfl_sync_400 = __shfl_sync(0xFFFFFFFF, sq_1, 16); + float _add_396; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_396) : "f"(_add_397), "f"(_shfl_sync_400)); + float _shfl_sync_401 = __shfl_sync(0xFFFFFFFF, sq_1, 24); float dot_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_364), "f"(_shfl_sync_369)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_396), "f"(_shfl_sync_401)); float o_1[4]; - float _fma_371; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_371) : "f"(y_1[0]), "f"(dot_1), "f"(nz_0)); - float _sub_370; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_370) : "f"(gq_1[0]), "f"(_fma_371)); - o_1[0] = mul_f32_rn_noftz(rq_1[0], _sub_370); - float _fma_373; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_373) : "f"(y_1[1]), "f"(dot_1), "f"(nz_0)); - float _sub_372; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_372) : "f"(gq_1[1]), "f"(_fma_373)); - o_1[1] = mul_f32_rn_noftz(rq_1[0], _sub_372); - float _fma_375; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_375) : "f"(y_1[2]), "f"(dot_1), "f"(nz_0)); - float _sub_374; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_374) : "f"(gq_1[2]), "f"(_fma_375)); - o_1[2] = mul_f32_rn_noftz(rq_1[0], _sub_374); - float _fma_377; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_377) : "f"(y_1[3]), "f"(dot_1), "f"(nz_0)); - float _sub_376; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_376) : "f"(gq_1[3]), "f"(_fma_377)); - o_1[3] = mul_f32_rn_noftz(rq_1[0], _sub_376); + float _fma_403; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_403) : "f"(y_1[0]), "f"(dot_1), "f"(nz_0)); + float _sub_402; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_402) : "f"(gq_1[0]), "f"(_fma_403)); + o_1[0] = mul_f32_rn_noftz(rq_1[0], _sub_402); + float _fma_405; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_405) : "f"(y_1[1]), "f"(dot_1), "f"(nz_0)); + float _sub_404; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_404) : "f"(gq_1[1]), "f"(_fma_405)); + o_1[1] = mul_f32_rn_noftz(rq_1[0], _sub_404); + float _fma_407; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_407) : "f"(y_1[2]), "f"(dot_1), "f"(nz_0)); + float _sub_406; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_406) : "f"(gq_1[2]), "f"(_fma_407)); + o_1[2] = mul_f32_rn_noftz(rq_1[0], _sub_406); + float _fma_409; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_409) : "f"(y_1[3]), "f"(dot_1), "f"(nz_0)); + float _sub_408; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_408) : "f"(gq_1[3]), "f"(_fma_409)); + o_1[3] = mul_f32_rn_noftz(rq_1[0], _sub_408); unsigned int ob_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -4128,106 +4326,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_1_1[4]; float p_1_1[4]; y_1_1[0] = mul_f32_rn_noftz(xk_1[0], rk_1[0]); - float _fma_378; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_378) : "f"(gk_1[0]), "f"(y_1_1[0]), "f"(nz_0)); - p_1_1[0] = _fma_378; + float _fma_410; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_410) : "f"(gk_1[0]), "f"(y_1_1[0]), "f"(nz_0)); + p_1_1[0] = _fma_410; y_1_1[1] = mul_f32_rn_noftz(xk_1[1], rk_1[0]); - float _fma_379; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_379) : "f"(gk_1[1]), "f"(y_1_1[1]), "f"(nz_0)); - p_1_1[1] = _fma_379; + float _fma_411; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_411) : "f"(gk_1[1]), "f"(y_1_1[1]), "f"(nz_0)); + p_1_1[1] = _fma_411; y_1_1[2] = mul_f32_rn_noftz(xk_1[2], rk_1[0]); - float _fma_380; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_380) : "f"(gk_1[2]), "f"(y_1_1[2]), "f"(nz_0)); - p_1_1[2] = _fma_380; + float _fma_412; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_412) : "f"(gk_1[2]), "f"(y_1_1[2]), "f"(nz_0)); + p_1_1[2] = _fma_412; y_1_1[3] = mul_f32_rn_noftz(xk_1[3], rk_1[0]); - float _fma_381; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_381) : "f"(gk_1[3]), "f"(y_1_1[3]), "f"(nz_0)); - p_1_1[3] = _fma_381; - float _shfl_xor_sync_383 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 4); - float _add_382; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_382) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_383)); - p_1_1[0] = _add_382; - float _shfl_xor_sync_385 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 4); - float _add_384; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_384) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_385)); - p_1_1[1] = _add_384; - float _shfl_xor_sync_387 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 4); - float _add_386; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_386) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_387)); - p_1_1[2] = _add_386; - float _shfl_xor_sync_389 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 4); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_389)); - p_1_1[3] = _add_388; - float _shfl_xor_sync_391 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 2); - float _add_390; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_390) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_391)); - p_1_1[0] = _add_390; - float _shfl_xor_sync_393 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 2); - float _add_392; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_393)); - p_1_1[1] = _add_392; - float _shfl_xor_sync_395 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 2); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_395)); - p_1_1[2] = _add_394; - float _shfl_xor_sync_397 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 2); - float _add_396; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_396) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_397)); - p_1_1[3] = _add_396; - float _shfl_xor_sync_399 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 1); - float _add_398; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_399)); - p_1_1[0] = _add_398; - float _shfl_xor_sync_401 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 1); - float _add_400; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_400) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_401)); - p_1_1[1] = _add_400; - float _shfl_xor_sync_403 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 1); - float _add_402; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_402) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_403)); - p_1_1[2] = _add_402; - float _shfl_xor_sync_405 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 1); - float _add_404; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_404) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_405)); - p_1_1[3] = _add_404; - float _add_406; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_406) : "f"(p_1_1[0]), "f"(p_1_1[2])); - float _add_407; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_407) : "f"(p_1_1[1]), "f"(p_1_1[3])); + float _fma_413; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_413) : "f"(gk_1[3]), "f"(y_1_1[3]), "f"(nz_0)); + p_1_1[3] = _fma_413; + float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 4); + float _add_414; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_414) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_415)); + p_1_1[0] = _add_414; + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 4); + float _add_416; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_416) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_417)); + p_1_1[1] = _add_416; + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 4); + float _add_418; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_418) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_419)); + p_1_1[2] = _add_418; + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 4); + float _add_420; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_420) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_421)); + p_1_1[3] = _add_420; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 2); + float _add_422; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_422) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_423)); + p_1_1[0] = _add_422; + float _shfl_xor_sync_425 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 2); + float _add_424; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_424) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_425)); + p_1_1[1] = _add_424; + float _shfl_xor_sync_427 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 2); + float _add_426; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_426) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_427)); + p_1_1[2] = _add_426; + float _shfl_xor_sync_429 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 2); + float _add_428; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_428) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_429)); + p_1_1[3] = _add_428; + float _shfl_xor_sync_431 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 1); + float _add_430; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_430) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_431)); + p_1_1[0] = _add_430; + float _shfl_xor_sync_433 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 1); + float _add_432; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_432) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_433)); + p_1_1[1] = _add_432; + float _shfl_xor_sync_435 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 1); + float _add_434; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_434) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_435)); + p_1_1[2] = _add_434; + float _shfl_xor_sync_437 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 1); + float _add_436; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_436) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_437)); + p_1_1[3] = _add_436; + float _add_438; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_438) : "f"(p_1_1[0]), "f"(p_1_1[2])); + float _add_439; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_439) : "f"(p_1_1[1]), "f"(p_1_1[3])); float sq_1_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1_1) : "f"(_add_406), "f"(_add_407)); - float _shfl_sync_410 = __shfl_sync(0xFFFFFFFF, sq_1_1, 0); - float _shfl_sync_411 = __shfl_sync(0xFFFFFFFF, sq_1_1, 8); - float _add_409; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_409) : "f"(_shfl_sync_410), "f"(_shfl_sync_411)); - float _shfl_sync_412 = __shfl_sync(0xFFFFFFFF, sq_1_1, 16); - float _add_408; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_408) : "f"(_add_409), "f"(_shfl_sync_412)); - float _shfl_sync_413 = __shfl_sync(0xFFFFFFFF, sq_1_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1_1) : "f"(_add_438), "f"(_add_439)); + float _shfl_sync_442 = __shfl_sync(0xFFFFFFFF, sq_1_1, 0); + float _shfl_sync_443 = __shfl_sync(0xFFFFFFFF, sq_1_1, 8); + float _add_441; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_441) : "f"(_shfl_sync_442), "f"(_shfl_sync_443)); + float _shfl_sync_444 = __shfl_sync(0xFFFFFFFF, sq_1_1, 16); + float _add_440; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_440) : "f"(_add_441), "f"(_shfl_sync_444)); + float _shfl_sync_445 = __shfl_sync(0xFFFFFFFF, sq_1_1, 24); float dot_1_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1_1) : "f"(_add_408), "f"(_shfl_sync_413)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1_1) : "f"(_add_440), "f"(_shfl_sync_445)); float o_1_1[4]; - float _fma_415; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_415) : "f"(y_1_1[0]), "f"(dot_1_1), "f"(nz_0)); - float _sub_414; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_414) : "f"(gk_1[0]), "f"(_fma_415)); - o_1_1[0] = mul_f32_rn_noftz(rk_1[0], _sub_414); - float _fma_417; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_417) : "f"(y_1_1[1]), "f"(dot_1_1), "f"(nz_0)); - float _sub_416; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_416) : "f"(gk_1[1]), "f"(_fma_417)); - o_1_1[1] = mul_f32_rn_noftz(rk_1[0], _sub_416); - float _fma_419; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_419) : "f"(y_1_1[2]), "f"(dot_1_1), "f"(nz_0)); - float _sub_418; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_418) : "f"(gk_1[2]), "f"(_fma_419)); - o_1_1[2] = mul_f32_rn_noftz(rk_1[0], _sub_418); - float _fma_421; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_421) : "f"(y_1_1[3]), "f"(dot_1_1), "f"(nz_0)); - float _sub_420; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_420) : "f"(gk_1[3]), "f"(_fma_421)); - o_1_1[3] = mul_f32_rn_noftz(rk_1[0], _sub_420); + float _fma_447; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_447) : "f"(y_1_1[0]), "f"(dot_1_1), "f"(nz_0)); + float _sub_446; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_446) : "f"(gk_1[0]), "f"(_fma_447)); + o_1_1[0] = mul_f32_rn_noftz(rk_1[0], _sub_446); + float _fma_449; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_449) : "f"(y_1_1[1]), "f"(dot_1_1), "f"(nz_0)); + float _sub_448; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_448) : "f"(gk_1[1]), "f"(_fma_449)); + o_1_1[1] = mul_f32_rn_noftz(rk_1[0], _sub_448); + float _fma_451; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_451) : "f"(y_1_1[2]), "f"(dot_1_1), "f"(nz_0)); + float _sub_450; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_450) : "f"(gk_1[2]), "f"(_fma_451)); + o_1_1[2] = mul_f32_rn_noftz(rk_1[0], _sub_450); + float _fma_453; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_453) : "f"(y_1_1[3]), "f"(dot_1_1), "f"(nz_0)); + float _sub_452; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_452) : "f"(gk_1[3]), "f"(_fma_453)); + o_1_1[3] = mul_f32_rn_noftz(rk_1[0], _sub_452); unsigned int ob_1_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -4237,8 +4435,8 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM asm volatile("{ .reg .pred p_; setp.ne.b32 p_, %3, 0; @p_ st.global.v2.b32 [%0], {%1, %2}; }" :: "l"(dk_ext + dx_1), "r"(ob_1_1[(0) + 0]), "r"(ob_1_1[(0) + 1]), "r"((unsigned int)(live_1)) : "memory"); } { - uint32_t _addr_422 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_422), "f"(tot_1228) : "memory"); + uint32_t _addr_454 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_454), "f"(tot_1228) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4267,9 +4465,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_0 = offset + _tmem_load_17[0]; if (t_9 < clen) { { - auto _gst_423 = dg_base + goff_1_0; - __builtin_assume(__isGlobal(_gst_423)); - *(reinterpret_cast(_gst_423) + (0)) = dg_1_0; + auto _gst_455 = dg_base + goff_1_0; + __builtin_assume(__isGlobal(_gst_455)); + *(reinterpret_cast(_gst_455) + (0)) = dg_1_0; } } int t_10 = 48 + 8 * th + 1; @@ -4277,9 +4475,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_1 = offset + _tmem_load_17[1]; if (t_10 < clen) { { - auto _gst_424 = dg_base + goff_1_1; - __builtin_assume(__isGlobal(_gst_424)); - *(reinterpret_cast(_gst_424) + (0)) = dg_1_1; + auto _gst_456 = dg_base + goff_1_1; + __builtin_assume(__isGlobal(_gst_456)); + *(reinterpret_cast(_gst_456) + (0)) = dg_1_1; } } int t_11 = 48 + 8 * th + 2; @@ -4287,9 +4485,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_2 = offset + _tmem_load_17[2]; if (t_11 < clen) { { - auto _gst_425 = dg_base + goff_1_2; - __builtin_assume(__isGlobal(_gst_425)); - *(reinterpret_cast(_gst_425) + (0)) = dg_1_2; + auto _gst_457 = dg_base + goff_1_2; + __builtin_assume(__isGlobal(_gst_457)); + *(reinterpret_cast(_gst_457) + (0)) = dg_1_2; } } int t_12 = 48 + 8 * th + 3; @@ -4297,9 +4495,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_3 = offset + _tmem_load_17[3]; if (t_12 < clen) { { - auto _gst_426 = dg_base + goff_1_3; - __builtin_assume(__isGlobal(_gst_426)); - *(reinterpret_cast(_gst_426) + (0)) = dg_1_3; + auto _gst_458 = dg_base + goff_1_3; + __builtin_assume(__isGlobal(_gst_458)); + *(reinterpret_cast(_gst_458) + (0)) = dg_1_3; } } int t_13 = 48 + 8 * th + 4; @@ -4307,9 +4505,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_4 = offset + _tmem_load_17[4]; if (t_13 < clen) { { - auto _gst_427 = dg_base + goff_1_4; - __builtin_assume(__isGlobal(_gst_427)); - *(reinterpret_cast(_gst_427) + (0)) = dg_1_4; + auto _gst_459 = dg_base + goff_1_4; + __builtin_assume(__isGlobal(_gst_459)); + *(reinterpret_cast(_gst_459) + (0)) = dg_1_4; } } int t_14 = 48 + 8 * th + 5; @@ -4317,9 +4515,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_5 = offset + _tmem_load_17[5]; if (t_14 < clen) { { - auto _gst_428 = dg_base + goff_1_5; - __builtin_assume(__isGlobal(_gst_428)); - *(reinterpret_cast(_gst_428) + (0)) = dg_1_5; + auto _gst_460 = dg_base + goff_1_5; + __builtin_assume(__isGlobal(_gst_460)); + *(reinterpret_cast(_gst_460) + (0)) = dg_1_5; } } int t_15 = 48 + 8 * th + 6; @@ -4327,9 +4525,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_6 = offset + _tmem_load_17[6]; if (t_15 < clen) { { - auto _gst_429 = dg_base + goff_1_6; - __builtin_assume(__isGlobal(_gst_429)); - *(reinterpret_cast(_gst_429) + (0)) = dg_1_6; + auto _gst_461 = dg_base + goff_1_6; + __builtin_assume(__isGlobal(_gst_461)); + *(reinterpret_cast(_gst_461) + (0)) = dg_1_6; } } int t_16 = 48 + 8 * th + 7; @@ -4337,9 +4535,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_7 = offset + _tmem_load_17[7]; if (t_16 < clen) { { - auto _gst_430 = dg_base + goff_1_7; - __builtin_assume(__isGlobal(_gst_430)); - *(reinterpret_cast(_gst_430) + (0)) = dg_1_7; + auto _gst_462 = dg_base + goff_1_7; + __builtin_assume(__isGlobal(_gst_462)); + *(reinterpret_cast(_gst_462) + (0)) = dg_1_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -4357,9 +4555,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_0 = offset_1250 + _tmem_load_19[0]; if (t_17 < clen) { { - auto _gst_431 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_431)); - *(reinterpret_cast(_gst_431) + (0)) = dg_0; + auto _gst_463 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_463)); + *(reinterpret_cast(_gst_463) + (0)) = dg_0; } } int t_18 = 32 + 8 * th + 1; @@ -4367,9 +4565,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1 = offset_1250 + _tmem_load_19[1]; if (t_18 < clen) { { - auto _gst_432 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_432)); - *(reinterpret_cast(_gst_432) + (0)) = dg_1; + auto _gst_464 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_464)); + *(reinterpret_cast(_gst_464) + (0)) = dg_1; } } int t_19 = 32 + 8 * th + 2; @@ -4377,9 +4575,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_2 = offset_1250 + _tmem_load_19[2]; if (t_19 < clen) { { - auto _gst_433 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_433)); - *(reinterpret_cast(_gst_433) + (0)) = dg_2; + auto _gst_465 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_465)); + *(reinterpret_cast(_gst_465) + (0)) = dg_2; } } int t_20 = 32 + 8 * th + 3; @@ -4387,9 +4585,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_3 = offset_1250 + _tmem_load_19[3]; if (t_20 < clen) { { - auto _gst_434 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_434)); - *(reinterpret_cast(_gst_434) + (0)) = dg_3; + auto _gst_466 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_466)); + *(reinterpret_cast(_gst_466) + (0)) = dg_3; } } int t_21 = 32 + 8 * th + 4; @@ -4397,9 +4595,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_4 = offset_1250 + _tmem_load_19[4]; if (t_21 < clen) { { - auto _gst_435 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_435)); - *(reinterpret_cast(_gst_435) + (0)) = dg_4; + auto _gst_467 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_467)); + *(reinterpret_cast(_gst_467) + (0)) = dg_4; } } int t_22 = 32 + 8 * th + 5; @@ -4407,9 +4605,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_5 = offset_1250 + _tmem_load_19[5]; if (t_22 < clen) { { - auto _gst_436 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_436)); - *(reinterpret_cast(_gst_436) + (0)) = dg_5; + auto _gst_468 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_468)); + *(reinterpret_cast(_gst_468) + (0)) = dg_5; } } int t_23 = 32 + 8 * th + 6; @@ -4417,9 +4615,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_6 = offset_1250 + _tmem_load_19[6]; if (t_23 < clen) { { - auto _gst_437 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_437)); - *(reinterpret_cast(_gst_437) + (0)) = dg_6; + auto _gst_469 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_469)); + *(reinterpret_cast(_gst_469) + (0)) = dg_6; } } int t_24 = 32 + 8 * th + 7; @@ -4427,15 +4625,15 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_7 = offset_1250 + _tmem_load_19[7]; if (t_24 < clen) { { - auto _gst_438 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_438)); - *(reinterpret_cast(_gst_438) + (0)) = dg_7; + auto _gst_470 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_470)); + *(reinterpret_cast(_gst_470) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi_1248 + t_lo_1249; { - uint32_t _addr_439 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_439), "f"(off_acc[0]) : "memory"); + uint32_t _addr_471 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_471), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4495,7 +4693,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[7]; - float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, (dk2b_9 - _tmem_load_24[7]) * k_t_9) + dgf_9; + float qe_15 = q_t_1_16 * e_dg_9; + float kp_27 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_9; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_9) * _tmem_load_23[7]); + float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_24[7], k_t_9, cr_16)) + dgf_9; s_run_1302[0] = s_run_1302[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4527,7 +4730,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[6]; - float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, (dk2b_10 - _tmem_load_24[6]) * k_t_10) + dgf_10; + float qe_16 = q_t_1_17 * e_dg_10; + float kp_28 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_10; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_10) * _tmem_load_23[6]); + float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_24[6], k_t_10, cr_17)) + dgf_10; s_run_1302[0] = s_run_1302[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4559,7 +4767,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[5]; - float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, (dk2b_11 - _tmem_load_24[5]) * k_t_11) + dgf_11; + float qe_17 = q_t_1_18 * e_dg_11; + float kp_29 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_11; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_11) * _tmem_load_23[5]); + float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_24[5], k_t_11, cr_18)) + dgf_11; s_run_1302[0] = s_run_1302[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4591,7 +4804,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[4]; - float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, (dk2b_12 - _tmem_load_24[4]) * k_t_12) + dgf_12; + float qe_18 = q_t_1_19 * e_dg_12; + float kp_30 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_12; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_12) * _tmem_load_23[4]); + float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_24[4], k_t_12, cr_19)) + dgf_12; s_run_1302[0] = s_run_1302[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4623,7 +4841,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[3]; - float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, (dk2b_13 - _tmem_load_24[3]) * k_t_13) + dgf_13; + float qe_19 = q_t_1_20 * e_dg_13; + float kp_31 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_13; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_13) * _tmem_load_23[3]); + float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_24[3], k_t_13, cr_20)) + dgf_13; s_run_1302[0] = s_run_1302[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4655,7 +4878,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[2]; - float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, (dk2b_14 - _tmem_load_24[2]) * k_t_14) + dgf_14; + float qe_20 = q_t_1_21 * e_dg_14; + float kp_32 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_14; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_14) * _tmem_load_23[2]); + float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_24[2], k_t_14, cr_21)) + dgf_14; s_run_1302[0] = s_run_1302[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4687,7 +4915,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[1]; - float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, (dk2b_15 - _tmem_load_24[1]) * k_t_15) + dgf_15; + float qe_21 = q_t_1_22 * e_dg_15; + float kp_33 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_15; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_15) * _tmem_load_23[1]); + float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_24[1], k_t_15, cr_22)) + dgf_15; s_run_1302[0] = s_run_1302[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4719,7 +4952,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_24[0]; - float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, (dk2b_16 - _tmem_load_24[0]) * k_t_16) + dgf_16; + float qe_22 = q_t_1_23 * e_dg_16; + float kp_34 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_16; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_16) * _tmem_load_23[0]); + float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, __fmaf_rn(dk2b_16 - _tmem_load_24[0], k_t_16, cr_23)) + dgf_16; s_run_1302[0] = s_run_1302[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4734,40 +4972,40 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; float send_7 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_440 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_440; + float _shfl_xor_sync_472 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_472; float send_8 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_441 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_441; + float _shfl_xor_sync_473 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_473; float send_9 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_442 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_442; + float _shfl_xor_sync_474 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_474; float send_10 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_443 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_443; + float _shfl_xor_sync_475 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_475; float v2_1480[2]; float send_11 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_444 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_444; + float _shfl_xor_sync_476 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_476; float send_12 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_445 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_445; + float _shfl_xor_sync_477 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_477; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_446 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_446; - float _shfl_xor_sync_447 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_447; - float _shfl_xor_sync_448 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_448; + float _shfl_xor_sync_478 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_478; + float _shfl_xor_sync_479 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_479; + float _shfl_xor_sync_480 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_480; asm volatile("tcgen05.fence::before_thread_sync;"); int ew_2 = 81920 - 32768 * th >> 2; float nz_1 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); bool live0_1 = 16 + 8 * th + qd < clen; bool live1_1 = 16 + 8 * th + qd + 4 < clen; - int _min_449 = ((16 + 8 * th + qd) < (clen - 1) ? (16 + 8 * th + qd) : (clen - 1)); - long long rowa_1 = ext0 + (long long)_min_449; - int _min_450 = ((16 + 8 * th + qd + 4) < (clen - 1) ? (16 + 8 * th + qd + 4) : (clen - 1)); - long long rowb_1 = ext0 + (long long)_min_450; + int _min_481 = ((16 + 8 * th + qd) < (clen - 1) ? (16 + 8 * th + qd) : (clen - 1)); + long long rowa_1 = ext0 + (long long)_min_481; + int _min_482 = ((16 + 8 * th + qd + 4) < (clen - 1) ? (16 + 8 * th + qd + 4) : (clen - 1)); + long long rowb_1 = ext0 + (long long)_min_482; float xqa_1[4]; float xka_1[4]; float xqb_1[4]; @@ -4795,52 +5033,52 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (ew_2 + 1024 + 128 * (qd + 4) + 4 * lane_1)))); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); { - uint2 _vld_451; - _vld_451 = *reinterpret_cast(q_raw + (rowa_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_451 = reinterpret_cast(&_vld_451); + uint2 _vld_483; + _vld_483 = *reinterpret_cast(q_raw + (rowa_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_483 = reinterpret_cast(&_vld_483); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqa_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_451[_pair]) << 16); - (&xqa_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_451[_pair]) & 0xffff0000u); + (&xqa_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_483[_pair]) << 16); + (&xqa_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_483[_pair]) & 0xffff0000u); } } { rqa_1[0] = *reinterpret_cast(qk_rstd + ((0 + rowa_1) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_452; - _vld_452 = *reinterpret_cast(k_raw + (rowa_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_452 = reinterpret_cast(&_vld_452); + uint2 _vld_484; + _vld_484 = *reinterpret_cast(k_raw + (rowa_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_484 = reinterpret_cast(&_vld_484); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xka_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_452[_pair]) << 16); - (&xka_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_452[_pair]) & 0xffff0000u); + (&xka_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_484[_pair]) << 16); + (&xka_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_484[_pair]) & 0xffff0000u); } } { rka_1[0] = *reinterpret_cast(qk_rstd + (((long long)num_tokens + rowa_1) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_453; - _vld_453 = *reinterpret_cast(q_raw + (rowb_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_453 = reinterpret_cast(&_vld_453); + uint2 _vld_485; + _vld_485 = *reinterpret_cast(q_raw + (rowb_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_485 = reinterpret_cast(&_vld_485); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqb_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_453[_pair]) << 16); - (&xqb_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_453[_pair]) & 0xffff0000u); + (&xqb_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_485[_pair]) << 16); + (&xqb_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_485[_pair]) & 0xffff0000u); } } { rqb_1[0] = *reinterpret_cast(qk_rstd + ((0 + rowb_1) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_454; - _vld_454 = *reinterpret_cast(k_raw + (rowb_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_454 = reinterpret_cast(&_vld_454); + uint2 _vld_486; + _vld_486 = *reinterpret_cast(k_raw + (rowb_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_486 = reinterpret_cast(&_vld_486); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xkb_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_454[_pair]) << 16); - (&xkb_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_454[_pair]) & 0xffff0000u); + (&xkb_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_486[_pair]) << 16); + (&xkb_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_486[_pair]) & 0xffff0000u); } } { @@ -4880,106 +5118,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_2[4]; float p_2[4]; y_2[0] = mul_f32_rn_noftz(xq_2[0], rq_2[0]); - float _fma_455; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_455) : "f"(gq_2[0]), "f"(y_2[0]), "f"(nz_1)); - p_2[0] = _fma_455; + float _fma_487; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_487) : "f"(gq_2[0]), "f"(y_2[0]), "f"(nz_1)); + p_2[0] = _fma_487; y_2[1] = mul_f32_rn_noftz(xq_2[1], rq_2[0]); - float _fma_456; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_456) : "f"(gq_2[1]), "f"(y_2[1]), "f"(nz_1)); - p_2[1] = _fma_456; + float _fma_488; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_488) : "f"(gq_2[1]), "f"(y_2[1]), "f"(nz_1)); + p_2[1] = _fma_488; y_2[2] = mul_f32_rn_noftz(xq_2[2], rq_2[0]); - float _fma_457; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_457) : "f"(gq_2[2]), "f"(y_2[2]), "f"(nz_1)); - p_2[2] = _fma_457; + float _fma_489; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_489) : "f"(gq_2[2]), "f"(y_2[2]), "f"(nz_1)); + p_2[2] = _fma_489; y_2[3] = mul_f32_rn_noftz(xq_2[3], rq_2[0]); - float _fma_458; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_458) : "f"(gq_2[3]), "f"(y_2[3]), "f"(nz_1)); - p_2[3] = _fma_458; - float _shfl_xor_sync_460 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 4); - float _add_459; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_459) : "f"(p_2[0]), "f"(_shfl_xor_sync_460)); - p_2[0] = _add_459; - float _shfl_xor_sync_462 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 4); - float _add_461; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_461) : "f"(p_2[1]), "f"(_shfl_xor_sync_462)); - p_2[1] = _add_461; - float _shfl_xor_sync_464 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 4); - float _add_463; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_463) : "f"(p_2[2]), "f"(_shfl_xor_sync_464)); - p_2[2] = _add_463; - float _shfl_xor_sync_466 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 4); - float _add_465; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_465) : "f"(p_2[3]), "f"(_shfl_xor_sync_466)); - p_2[3] = _add_465; - float _shfl_xor_sync_468 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 2); - float _add_467; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_467) : "f"(p_2[0]), "f"(_shfl_xor_sync_468)); - p_2[0] = _add_467; - float _shfl_xor_sync_470 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 2); - float _add_469; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_469) : "f"(p_2[1]), "f"(_shfl_xor_sync_470)); - p_2[1] = _add_469; - float _shfl_xor_sync_472 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 2); - float _add_471; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_471) : "f"(p_2[2]), "f"(_shfl_xor_sync_472)); - p_2[2] = _add_471; - float _shfl_xor_sync_474 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 2); - float _add_473; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_473) : "f"(p_2[3]), "f"(_shfl_xor_sync_474)); - p_2[3] = _add_473; - float _shfl_xor_sync_476 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 1); - float _add_475; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_475) : "f"(p_2[0]), "f"(_shfl_xor_sync_476)); - p_2[0] = _add_475; - float _shfl_xor_sync_478 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 1); - float _add_477; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_477) : "f"(p_2[1]), "f"(_shfl_xor_sync_478)); - p_2[1] = _add_477; - float _shfl_xor_sync_480 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 1); - float _add_479; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_479) : "f"(p_2[2]), "f"(_shfl_xor_sync_480)); - p_2[2] = _add_479; - float _shfl_xor_sync_482 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 1); - float _add_481; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_481) : "f"(p_2[3]), "f"(_shfl_xor_sync_482)); - p_2[3] = _add_481; - float _add_483; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_483) : "f"(p_2[0]), "f"(p_2[2])); - float _add_484; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_484) : "f"(p_2[1]), "f"(p_2[3])); + float _fma_490; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_490) : "f"(gq_2[3]), "f"(y_2[3]), "f"(nz_1)); + p_2[3] = _fma_490; + float _shfl_xor_sync_492 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 4); + float _add_491; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_491) : "f"(p_2[0]), "f"(_shfl_xor_sync_492)); + p_2[0] = _add_491; + float _shfl_xor_sync_494 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 4); + float _add_493; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_493) : "f"(p_2[1]), "f"(_shfl_xor_sync_494)); + p_2[1] = _add_493; + float _shfl_xor_sync_496 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 4); + float _add_495; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_495) : "f"(p_2[2]), "f"(_shfl_xor_sync_496)); + p_2[2] = _add_495; + float _shfl_xor_sync_498 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 4); + float _add_497; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_497) : "f"(p_2[3]), "f"(_shfl_xor_sync_498)); + p_2[3] = _add_497; + float _shfl_xor_sync_500 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 2); + float _add_499; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_499) : "f"(p_2[0]), "f"(_shfl_xor_sync_500)); + p_2[0] = _add_499; + float _shfl_xor_sync_502 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 2); + float _add_501; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_501) : "f"(p_2[1]), "f"(_shfl_xor_sync_502)); + p_2[1] = _add_501; + float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 2); + float _add_503; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_503) : "f"(p_2[2]), "f"(_shfl_xor_sync_504)); + p_2[2] = _add_503; + float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 2); + float _add_505; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_505) : "f"(p_2[3]), "f"(_shfl_xor_sync_506)); + p_2[3] = _add_505; + float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 1); + float _add_507; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_507) : "f"(p_2[0]), "f"(_shfl_xor_sync_508)); + p_2[0] = _add_507; + float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 1); + float _add_509; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_509) : "f"(p_2[1]), "f"(_shfl_xor_sync_510)); + p_2[1] = _add_509; + float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 1); + float _add_511; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_511) : "f"(p_2[2]), "f"(_shfl_xor_sync_512)); + p_2[2] = _add_511; + float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 1); + float _add_513; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_513) : "f"(p_2[3]), "f"(_shfl_xor_sync_514)); + p_2[3] = _add_513; + float _add_515; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_515) : "f"(p_2[0]), "f"(p_2[2])); + float _add_516; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_516) : "f"(p_2[1]), "f"(p_2[3])); float sq_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2) : "f"(_add_483), "f"(_add_484)); - float _shfl_sync_487 = __shfl_sync(0xFFFFFFFF, sq_2, 0); - float _shfl_sync_488 = __shfl_sync(0xFFFFFFFF, sq_2, 8); - float _add_486; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_486) : "f"(_shfl_sync_487), "f"(_shfl_sync_488)); - float _shfl_sync_489 = __shfl_sync(0xFFFFFFFF, sq_2, 16); - float _add_485; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_485) : "f"(_add_486), "f"(_shfl_sync_489)); - float _shfl_sync_490 = __shfl_sync(0xFFFFFFFF, sq_2, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2) : "f"(_add_515), "f"(_add_516)); + float _shfl_sync_519 = __shfl_sync(0xFFFFFFFF, sq_2, 0); + float _shfl_sync_520 = __shfl_sync(0xFFFFFFFF, sq_2, 8); + float _add_518; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_518) : "f"(_shfl_sync_519), "f"(_shfl_sync_520)); + float _shfl_sync_521 = __shfl_sync(0xFFFFFFFF, sq_2, 16); + float _add_517; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_517) : "f"(_add_518), "f"(_shfl_sync_521)); + float _shfl_sync_522 = __shfl_sync(0xFFFFFFFF, sq_2, 24); float dot_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_485), "f"(_shfl_sync_490)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_517), "f"(_shfl_sync_522)); float o_2[4]; - float _fma_492; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_492) : "f"(y_2[0]), "f"(dot_2), "f"(nz_1)); - float _sub_491; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_491) : "f"(gq_2[0]), "f"(_fma_492)); - o_2[0] = mul_f32_rn_noftz(rq_2[0], _sub_491); - float _fma_494; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_494) : "f"(y_2[1]), "f"(dot_2), "f"(nz_1)); - float _sub_493; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_493) : "f"(gq_2[1]), "f"(_fma_494)); - o_2[1] = mul_f32_rn_noftz(rq_2[0], _sub_493); - float _fma_496; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_496) : "f"(y_2[2]), "f"(dot_2), "f"(nz_1)); - float _sub_495; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_495) : "f"(gq_2[2]), "f"(_fma_496)); - o_2[2] = mul_f32_rn_noftz(rq_2[0], _sub_495); - float _fma_498; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_498) : "f"(y_2[3]), "f"(dot_2), "f"(nz_1)); - float _sub_497; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_497) : "f"(gq_2[3]), "f"(_fma_498)); - o_2[3] = mul_f32_rn_noftz(rq_2[0], _sub_497); + float _fma_524; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_524) : "f"(y_2[0]), "f"(dot_2), "f"(nz_1)); + float _sub_523; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_523) : "f"(gq_2[0]), "f"(_fma_524)); + o_2[0] = mul_f32_rn_noftz(rq_2[0], _sub_523); + float _fma_526; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_526) : "f"(y_2[1]), "f"(dot_2), "f"(nz_1)); + float _sub_525; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_525) : "f"(gq_2[1]), "f"(_fma_526)); + o_2[1] = mul_f32_rn_noftz(rq_2[0], _sub_525); + float _fma_528; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_528) : "f"(y_2[2]), "f"(dot_2), "f"(nz_1)); + float _sub_527; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_527) : "f"(gq_2[2]), "f"(_fma_528)); + o_2[2] = mul_f32_rn_noftz(rq_2[0], _sub_527); + float _fma_530; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_530) : "f"(y_2[3]), "f"(dot_2), "f"(nz_1)); + float _sub_529; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_529) : "f"(gq_2[3]), "f"(_fma_530)); + o_2[3] = mul_f32_rn_noftz(rq_2[0], _sub_529); unsigned int ob_2[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -4990,106 +5228,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_2_1[4]; float p_2_1[4]; y_2_1[0] = mul_f32_rn_noftz(xk_2[0], rk_2[0]); - float _fma_499; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_499) : "f"(gk_2[0]), "f"(y_2_1[0]), "f"(nz_1)); - p_2_1[0] = _fma_499; + float _fma_531; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_531) : "f"(gk_2[0]), "f"(y_2_1[0]), "f"(nz_1)); + p_2_1[0] = _fma_531; y_2_1[1] = mul_f32_rn_noftz(xk_2[1], rk_2[0]); - float _fma_500; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_500) : "f"(gk_2[1]), "f"(y_2_1[1]), "f"(nz_1)); - p_2_1[1] = _fma_500; + float _fma_532; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_532) : "f"(gk_2[1]), "f"(y_2_1[1]), "f"(nz_1)); + p_2_1[1] = _fma_532; y_2_1[2] = mul_f32_rn_noftz(xk_2[2], rk_2[0]); - float _fma_501; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_501) : "f"(gk_2[2]), "f"(y_2_1[2]), "f"(nz_1)); - p_2_1[2] = _fma_501; + float _fma_533; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_533) : "f"(gk_2[2]), "f"(y_2_1[2]), "f"(nz_1)); + p_2_1[2] = _fma_533; y_2_1[3] = mul_f32_rn_noftz(xk_2[3], rk_2[0]); - float _fma_502; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_502) : "f"(gk_2[3]), "f"(y_2_1[3]), "f"(nz_1)); - p_2_1[3] = _fma_502; - float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 4); - float _add_503; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_503) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_504)); - p_2_1[0] = _add_503; - float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 4); - float _add_505; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_505) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_506)); - p_2_1[1] = _add_505; - float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 4); - float _add_507; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_507) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_508)); - p_2_1[2] = _add_507; - float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 4); - float _add_509; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_509) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_510)); - p_2_1[3] = _add_509; - float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 2); - float _add_511; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_511) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_512)); - p_2_1[0] = _add_511; - float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 2); - float _add_513; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_513) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_514)); - p_2_1[1] = _add_513; - float _shfl_xor_sync_516 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 2); - float _add_515; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_515) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_516)); - p_2_1[2] = _add_515; - float _shfl_xor_sync_518 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 2); - float _add_517; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_517) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_518)); - p_2_1[3] = _add_517; - float _shfl_xor_sync_520 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 1); - float _add_519; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_519) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_520)); - p_2_1[0] = _add_519; - float _shfl_xor_sync_522 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 1); - float _add_521; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_521) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_522)); - p_2_1[1] = _add_521; - float _shfl_xor_sync_524 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 1); - float _add_523; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_523) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_524)); - p_2_1[2] = _add_523; - float _shfl_xor_sync_526 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 1); - float _add_525; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_525) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_526)); - p_2_1[3] = _add_525; - float _add_527; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_527) : "f"(p_2_1[0]), "f"(p_2_1[2])); - float _add_528; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_528) : "f"(p_2_1[1]), "f"(p_2_1[3])); + float _fma_534; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_534) : "f"(gk_2[3]), "f"(y_2_1[3]), "f"(nz_1)); + p_2_1[3] = _fma_534; + float _shfl_xor_sync_536 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 4); + float _add_535; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_535) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_536)); + p_2_1[0] = _add_535; + float _shfl_xor_sync_538 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 4); + float _add_537; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_537) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_538)); + p_2_1[1] = _add_537; + float _shfl_xor_sync_540 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 4); + float _add_539; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_539) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_540)); + p_2_1[2] = _add_539; + float _shfl_xor_sync_542 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 4); + float _add_541; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_541) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_542)); + p_2_1[3] = _add_541; + float _shfl_xor_sync_544 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 2); + float _add_543; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_543) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_544)); + p_2_1[0] = _add_543; + float _shfl_xor_sync_546 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 2); + float _add_545; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_545) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_546)); + p_2_1[1] = _add_545; + float _shfl_xor_sync_548 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 2); + float _add_547; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_547) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_548)); + p_2_1[2] = _add_547; + float _shfl_xor_sync_550 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 2); + float _add_549; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_549) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_550)); + p_2_1[3] = _add_549; + float _shfl_xor_sync_552 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 1); + float _add_551; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_551) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_552)); + p_2_1[0] = _add_551; + float _shfl_xor_sync_554 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 1); + float _add_553; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_553) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_554)); + p_2_1[1] = _add_553; + float _shfl_xor_sync_556 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 1); + float _add_555; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_555) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_556)); + p_2_1[2] = _add_555; + float _shfl_xor_sync_558 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 1); + float _add_557; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_557) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_558)); + p_2_1[3] = _add_557; + float _add_559; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_559) : "f"(p_2_1[0]), "f"(p_2_1[2])); + float _add_560; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_560) : "f"(p_2_1[1]), "f"(p_2_1[3])); float sq_2_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2_1) : "f"(_add_527), "f"(_add_528)); - float _shfl_sync_531 = __shfl_sync(0xFFFFFFFF, sq_2_1, 0); - float _shfl_sync_532 = __shfl_sync(0xFFFFFFFF, sq_2_1, 8); - float _add_530; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_530) : "f"(_shfl_sync_531), "f"(_shfl_sync_532)); - float _shfl_sync_533 = __shfl_sync(0xFFFFFFFF, sq_2_1, 16); - float _add_529; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_529) : "f"(_add_530), "f"(_shfl_sync_533)); - float _shfl_sync_534 = __shfl_sync(0xFFFFFFFF, sq_2_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2_1) : "f"(_add_559), "f"(_add_560)); + float _shfl_sync_563 = __shfl_sync(0xFFFFFFFF, sq_2_1, 0); + float _shfl_sync_564 = __shfl_sync(0xFFFFFFFF, sq_2_1, 8); + float _add_562; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_562) : "f"(_shfl_sync_563), "f"(_shfl_sync_564)); + float _shfl_sync_565 = __shfl_sync(0xFFFFFFFF, sq_2_1, 16); + float _add_561; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_561) : "f"(_add_562), "f"(_shfl_sync_565)); + float _shfl_sync_566 = __shfl_sync(0xFFFFFFFF, sq_2_1, 24); float dot_2_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2_1) : "f"(_add_529), "f"(_shfl_sync_534)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2_1) : "f"(_add_561), "f"(_shfl_sync_566)); float o_2_1[4]; - float _fma_536; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_536) : "f"(y_2_1[0]), "f"(dot_2_1), "f"(nz_1)); - float _sub_535; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_535) : "f"(gk_2[0]), "f"(_fma_536)); - o_2_1[0] = mul_f32_rn_noftz(rk_2[0], _sub_535); - float _fma_538; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_538) : "f"(y_2_1[1]), "f"(dot_2_1), "f"(nz_1)); - float _sub_537; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_537) : "f"(gk_2[1]), "f"(_fma_538)); - o_2_1[1] = mul_f32_rn_noftz(rk_2[0], _sub_537); - float _fma_540; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_540) : "f"(y_2_1[2]), "f"(dot_2_1), "f"(nz_1)); - float _sub_539; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_539) : "f"(gk_2[2]), "f"(_fma_540)); - o_2_1[2] = mul_f32_rn_noftz(rk_2[0], _sub_539); - float _fma_542; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_542) : "f"(y_2_1[3]), "f"(dot_2_1), "f"(nz_1)); - float _sub_541; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_541) : "f"(gk_2[3]), "f"(_fma_542)); - o_2_1[3] = mul_f32_rn_noftz(rk_2[0], _sub_541); + float _fma_568; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_568) : "f"(y_2_1[0]), "f"(dot_2_1), "f"(nz_1)); + float _sub_567; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_567) : "f"(gk_2[0]), "f"(_fma_568)); + o_2_1[0] = mul_f32_rn_noftz(rk_2[0], _sub_567); + float _fma_570; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_570) : "f"(y_2_1[1]), "f"(dot_2_1), "f"(nz_1)); + float _sub_569; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_569) : "f"(gk_2[1]), "f"(_fma_570)); + o_2_1[1] = mul_f32_rn_noftz(rk_2[0], _sub_569); + float _fma_572; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_572) : "f"(y_2_1[2]), "f"(dot_2_1), "f"(nz_1)); + float _sub_571; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(gk_2[2]), "f"(_fma_572)); + o_2_1[2] = mul_f32_rn_noftz(rk_2[0], _sub_571); + float _fma_574; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_574) : "f"(y_2_1[3]), "f"(dot_2_1), "f"(nz_1)); + float _sub_573; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_573) : "f"(gk_2[3]), "f"(_fma_574)); + o_2_1[3] = mul_f32_rn_noftz(rk_2[0], _sub_573); unsigned int ob_2_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -5100,8 +5338,8 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM } asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_543 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_543), "f"(tot_1471) : "memory"); + uint32_t _addr_575 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_575), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5130,9 +5368,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_8 = offset_1488 + _tmem_load_25[0]; if (t_33 < clen) { { - auto _gst_544 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_544)); - *(reinterpret_cast(_gst_544) + (0)) = dg_8; + auto _gst_576 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_576)); + *(reinterpret_cast(_gst_576) + (0)) = dg_8; } } int t_34 = 16 + 8 * th + 1; @@ -5140,9 +5378,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_9 = offset_1488 + _tmem_load_25[1]; if (t_34 < clen) { { - auto _gst_545 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_545)); - *(reinterpret_cast(_gst_545) + (0)) = dg_9; + auto _gst_577 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_577)); + *(reinterpret_cast(_gst_577) + (0)) = dg_9; } } int t_35 = 16 + 8 * th + 2; @@ -5150,9 +5388,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_10 = offset_1488 + _tmem_load_25[2]; if (t_35 < clen) { { - auto _gst_546 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_546)); - *(reinterpret_cast(_gst_546) + (0)) = dg_10; + auto _gst_578 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_578)); + *(reinterpret_cast(_gst_578) + (0)) = dg_10; } } int t_36 = 16 + 8 * th + 3; @@ -5160,9 +5398,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_11 = offset_1488 + _tmem_load_25[3]; if (t_36 < clen) { { - auto _gst_547 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_547)); - *(reinterpret_cast(_gst_547) + (0)) = dg_11; + auto _gst_579 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_579)); + *(reinterpret_cast(_gst_579) + (0)) = dg_11; } } int t_37 = 16 + 8 * th + 4; @@ -5170,9 +5408,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_12 = offset_1488 + _tmem_load_25[4]; if (t_37 < clen) { { - auto _gst_548 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_548)); - *(reinterpret_cast(_gst_548) + (0)) = dg_12; + auto _gst_580 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_580)); + *(reinterpret_cast(_gst_580) + (0)) = dg_12; } } int t_38 = 16 + 8 * th + 5; @@ -5180,9 +5418,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_13 = offset_1488 + _tmem_load_25[5]; if (t_38 < clen) { { - auto _gst_549 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_549)); - *(reinterpret_cast(_gst_549) + (0)) = dg_13; + auto _gst_581 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_581)); + *(reinterpret_cast(_gst_581) + (0)) = dg_13; } } int t_39 = 16 + 8 * th + 6; @@ -5190,9 +5428,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_14 = offset_1488 + _tmem_load_25[6]; if (t_39 < clen) { { - auto _gst_550 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_550)); - *(reinterpret_cast(_gst_550) + (0)) = dg_14; + auto _gst_582 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_582)); + *(reinterpret_cast(_gst_582) + (0)) = dg_14; } } int t_40 = 16 + 8 * th + 7; @@ -5200,15 +5438,15 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_15 = offset_1488 + _tmem_load_25[7]; if (t_40 < clen) { { - auto _gst_551 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_551)); - *(reinterpret_cast(_gst_551) + (0)) = dg_15; + auto _gst_583 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_583)); + *(reinterpret_cast(_gst_583) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_1486 + t_lo_1487; { - uint32_t _addr_552 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_552), "f"(off_acc[0]) : "memory"); + uint32_t _addr_584 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_584), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -5262,7 +5500,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[7]; - float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, (dk2b_17 - _tmem_load_28[7]) * k_t_17) + dgf_17; + float qe_23 = q_t_1_24 * e_dg_17; + float kp_35 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_35; + float kbe_23 = (float)kb_23 * e_dg_17; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_35 * e_dg_17) * _tmem_load_27[7]); + float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, __fmaf_rn(dk2b_17 - _tmem_load_28[7], k_t_17, cr_24)) + dgf_17; s_run_1524[0] = s_run_1524[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5294,7 +5537,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[6]; - float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, (dk2b_18 - _tmem_load_28[6]) * k_t_18) + dgf_18; + float qe_24 = q_t_1_25 * e_dg_18; + float kp_36 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_36; + float kbe_24 = (float)kb_24 * e_dg_18; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_36 * e_dg_18) * _tmem_load_27[6]); + float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, __fmaf_rn(dk2b_18 - _tmem_load_28[6], k_t_18, cr_25)) + dgf_18; s_run_1524[0] = s_run_1524[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5326,7 +5574,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[5]; - float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, (dk2b_19 - _tmem_load_28[5]) * k_t_19) + dgf_19; + float qe_25 = q_t_1_26 * e_dg_19; + float kp_37 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_37; + float kbe_25 = (float)kb_25 * e_dg_19; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_37 * e_dg_19) * _tmem_load_27[5]); + float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, __fmaf_rn(dk2b_19 - _tmem_load_28[5], k_t_19, cr_26)) + dgf_19; s_run_1524[0] = s_run_1524[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5358,7 +5611,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[4]; - float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, (dk2b_20 - _tmem_load_28[4]) * k_t_20) + dgf_20; + float qe_26 = q_t_1_27 * e_dg_20; + float kp_38 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_38; + float kbe_26 = (float)kb_26 * e_dg_20; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_38 * e_dg_20) * _tmem_load_27[4]); + float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, __fmaf_rn(dk2b_20 - _tmem_load_28[4], k_t_20, cr_27)) + dgf_20; s_run_1524[0] = s_run_1524[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5390,7 +5648,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[3]; - float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, (dk2b_21 - _tmem_load_28[3]) * k_t_21) + dgf_21; + float qe_27 = q_t_1_28 * e_dg_21; + float kp_39 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_39; + float kbe_27 = (float)kb_27 * e_dg_21; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_39 * e_dg_21) * _tmem_load_27[3]); + float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, __fmaf_rn(dk2b_21 - _tmem_load_28[3], k_t_21, cr_28)) + dgf_21; s_run_1524[0] = s_run_1524[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5422,7 +5685,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[2]; - float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, (dk2b_22 - _tmem_load_28[2]) * k_t_22) + dgf_22; + float qe_28 = q_t_1_29 * e_dg_22; + float kp_40 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_40; + float kbe_28 = (float)kb_28 * e_dg_22; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_40 * e_dg_22) * _tmem_load_27[2]); + float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, __fmaf_rn(dk2b_22 - _tmem_load_28[2], k_t_22, cr_29)) + dgf_22; s_run_1524[0] = s_run_1524[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5454,7 +5722,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[1]; - float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, (dk2b_23 - _tmem_load_28[1]) * k_t_23) + dgf_23; + float qe_29 = q_t_1_30 * e_dg_23; + float kp_41 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_41; + float kbe_29 = (float)kb_29 * e_dg_23; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_41 * e_dg_23) * _tmem_load_27[1]); + float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, __fmaf_rn(dk2b_23 - _tmem_load_28[1], k_t_23, cr_30)) + dgf_23; s_run_1524[0] = s_run_1524[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5486,7 +5759,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + _tmem_load_28[0]; - float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, (dk2b_24 - _tmem_load_28[0]) * k_t_24) + dgf_24; + float qe_30 = q_t_1_31 * e_dg_24; + float kp_42 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_42; + float kbe_30 = (float)kb_30 * e_dg_24; + float cr_31 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_42 * e_dg_24) * _tmem_load_27[0]); + float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, __fmaf_rn(dk2b_24 - _tmem_load_28[0], k_t_24, cr_31)) + dgf_24; s_run_1524[0] = s_run_1524[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5501,39 +5779,39 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; float send_13 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_553 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_553; + float _shfl_xor_sync_585 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_585; float send_14 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_554 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_554; + float _shfl_xor_sync_586 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_586; float send_15 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_555 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_555; + float _shfl_xor_sync_587 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_587; float send_16 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_556 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_556; + float _shfl_xor_sync_588 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_588; float v2_1702[2]; float send_17 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_557 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_557; + float _shfl_xor_sync_589 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_589; float send_18 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_558 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_558; + float _shfl_xor_sync_590 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_590; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_559 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_559; - float _shfl_xor_sync_560 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_560; - float _shfl_xor_sync_561 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_561; + float _shfl_xor_sync_591 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_591; + float _shfl_xor_sync_592 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_592; + float _shfl_xor_sync_593 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_593; int ew_3 = 98304 - 32768 * th >> 2; float nz_2 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); bool live0_2 = 8 * th + qd < clen; bool live1_2 = 8 * th + qd + 4 < clen; - int _min_562 = ((8 * th + qd) < (clen - 1) ? (8 * th + qd) : (clen - 1)); - long long rowa_2 = ext0 + (long long)_min_562; - int _min_563 = ((8 * th + qd + 4) < (clen - 1) ? (8 * th + qd + 4) : (clen - 1)); - long long rowb_2 = ext0 + (long long)_min_563; + int _min_594 = ((8 * th + qd) < (clen - 1) ? (8 * th + qd) : (clen - 1)); + long long rowa_2 = ext0 + (long long)_min_594; + int _min_595 = ((8 * th + qd + 4) < (clen - 1) ? (8 * th + qd + 4) : (clen - 1)); + long long rowb_2 = ext0 + (long long)_min_595; float xqa_2[4]; float xka_2[4]; float xqb_2[4]; @@ -5560,52 +5838,52 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM : "=r"(*reinterpret_cast(&gkb_2[0])), "=r"(*reinterpret_cast(&gkb_2[(0) + 1])), "=r"(*reinterpret_cast(&gkb_2[(0) + 2])), "=r"(*reinterpret_cast(&gkb_2[(0) + 3])) : "r"(pool_f32_addr + (unsigned int)(4 * (ew_3 + 1024 + 128 * (qd + 4) + 4 * lane_1)))); { - uint2 _vld_564; - _vld_564 = *reinterpret_cast(q_raw + (rowa_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_564 = reinterpret_cast(&_vld_564); + uint2 _vld_596; + _vld_596 = *reinterpret_cast(q_raw + (rowa_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_596 = reinterpret_cast(&_vld_596); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqa_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_564[_pair]) << 16); - (&xqa_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_564[_pair]) & 0xffff0000u); + (&xqa_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_596[_pair]) << 16); + (&xqa_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_596[_pair]) & 0xffff0000u); } } { rqa_2[0] = *reinterpret_cast(qk_rstd + ((0 + rowa_2) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_565; - _vld_565 = *reinterpret_cast(k_raw + (rowa_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_565 = reinterpret_cast(&_vld_565); + uint2 _vld_597; + _vld_597 = *reinterpret_cast(k_raw + (rowa_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_597 = reinterpret_cast(&_vld_597); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xka_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_565[_pair]) << 16); - (&xka_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_565[_pair]) & 0xffff0000u); + (&xka_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_597[_pair]) << 16); + (&xka_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_597[_pair]) & 0xffff0000u); } } { rka_2[0] = *reinterpret_cast(qk_rstd + (((long long)num_tokens + rowa_2) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_566; - _vld_566 = *reinterpret_cast(q_raw + (rowb_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_566 = reinterpret_cast(&_vld_566); + uint2 _vld_598; + _vld_598 = *reinterpret_cast(q_raw + (rowb_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_598 = reinterpret_cast(&_vld_598); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqb_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_566[_pair]) << 16); - (&xqb_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_566[_pair]) & 0xffff0000u); + (&xqb_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_598[_pair]) << 16); + (&xqb_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_598[_pair]) & 0xffff0000u); } } { rqb_2[0] = *reinterpret_cast(qk_rstd + ((0 + rowb_2) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_567; - _vld_567 = *reinterpret_cast(k_raw + (rowb_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_567 = reinterpret_cast(&_vld_567); + uint2 _vld_599; + _vld_599 = *reinterpret_cast(k_raw + (rowb_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_599 = reinterpret_cast(&_vld_599); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xkb_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_567[_pair]) << 16); - (&xkb_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_567[_pair]) & 0xffff0000u); + (&xkb_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_599[_pair]) << 16); + (&xkb_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_599[_pair]) & 0xffff0000u); } } { @@ -5645,106 +5923,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_3[4]; float p_3[4]; y_3[0] = mul_f32_rn_noftz(xq_3[0], rq_3[0]); - float _fma_568; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_568) : "f"(gq_3[0]), "f"(y_3[0]), "f"(nz_2)); - p_3[0] = _fma_568; + float _fma_600; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_600) : "f"(gq_3[0]), "f"(y_3[0]), "f"(nz_2)); + p_3[0] = _fma_600; y_3[1] = mul_f32_rn_noftz(xq_3[1], rq_3[0]); - float _fma_569; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_569) : "f"(gq_3[1]), "f"(y_3[1]), "f"(nz_2)); - p_3[1] = _fma_569; + float _fma_601; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_601) : "f"(gq_3[1]), "f"(y_3[1]), "f"(nz_2)); + p_3[1] = _fma_601; y_3[2] = mul_f32_rn_noftz(xq_3[2], rq_3[0]); - float _fma_570; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_570) : "f"(gq_3[2]), "f"(y_3[2]), "f"(nz_2)); - p_3[2] = _fma_570; + float _fma_602; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_602) : "f"(gq_3[2]), "f"(y_3[2]), "f"(nz_2)); + p_3[2] = _fma_602; y_3[3] = mul_f32_rn_noftz(xq_3[3], rq_3[0]); - float _fma_571; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_571) : "f"(gq_3[3]), "f"(y_3[3]), "f"(nz_2)); - p_3[3] = _fma_571; - float _shfl_xor_sync_573 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 4); - float _add_572; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_572) : "f"(p_3[0]), "f"(_shfl_xor_sync_573)); - p_3[0] = _add_572; - float _shfl_xor_sync_575 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 4); - float _add_574; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(p_3[1]), "f"(_shfl_xor_sync_575)); - p_3[1] = _add_574; - float _shfl_xor_sync_577 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 4); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(p_3[2]), "f"(_shfl_xor_sync_577)); - p_3[2] = _add_576; - float _shfl_xor_sync_579 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 4); - float _add_578; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_578) : "f"(p_3[3]), "f"(_shfl_xor_sync_579)); - p_3[3] = _add_578; - float _shfl_xor_sync_581 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 2); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(p_3[0]), "f"(_shfl_xor_sync_581)); - p_3[0] = _add_580; - float _shfl_xor_sync_583 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 2); - float _add_582; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_582) : "f"(p_3[1]), "f"(_shfl_xor_sync_583)); - p_3[1] = _add_582; - float _shfl_xor_sync_585 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 2); - float _add_584; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(p_3[2]), "f"(_shfl_xor_sync_585)); - p_3[2] = _add_584; - float _shfl_xor_sync_587 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 2); - float _add_586; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_586) : "f"(p_3[3]), "f"(_shfl_xor_sync_587)); - p_3[3] = _add_586; - float _shfl_xor_sync_589 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 1); - float _add_588; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_588) : "f"(p_3[0]), "f"(_shfl_xor_sync_589)); - p_3[0] = _add_588; - float _shfl_xor_sync_591 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 1); - float _add_590; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_590) : "f"(p_3[1]), "f"(_shfl_xor_sync_591)); - p_3[1] = _add_590; - float _shfl_xor_sync_593 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 1); - float _add_592; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_592) : "f"(p_3[2]), "f"(_shfl_xor_sync_593)); - p_3[2] = _add_592; - float _shfl_xor_sync_595 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 1); - float _add_594; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_594) : "f"(p_3[3]), "f"(_shfl_xor_sync_595)); - p_3[3] = _add_594; - float _add_596; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_596) : "f"(p_3[0]), "f"(p_3[2])); - float _add_597; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_597) : "f"(p_3[1]), "f"(p_3[3])); + float _fma_603; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_603) : "f"(gq_3[3]), "f"(y_3[3]), "f"(nz_2)); + p_3[3] = _fma_603; + float _shfl_xor_sync_605 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 4); + float _add_604; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_604) : "f"(p_3[0]), "f"(_shfl_xor_sync_605)); + p_3[0] = _add_604; + float _shfl_xor_sync_607 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 4); + float _add_606; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_606) : "f"(p_3[1]), "f"(_shfl_xor_sync_607)); + p_3[1] = _add_606; + float _shfl_xor_sync_609 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 4); + float _add_608; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_608) : "f"(p_3[2]), "f"(_shfl_xor_sync_609)); + p_3[2] = _add_608; + float _shfl_xor_sync_611 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 4); + float _add_610; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_610) : "f"(p_3[3]), "f"(_shfl_xor_sync_611)); + p_3[3] = _add_610; + float _shfl_xor_sync_613 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 2); + float _add_612; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_612) : "f"(p_3[0]), "f"(_shfl_xor_sync_613)); + p_3[0] = _add_612; + float _shfl_xor_sync_615 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 2); + float _add_614; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_614) : "f"(p_3[1]), "f"(_shfl_xor_sync_615)); + p_3[1] = _add_614; + float _shfl_xor_sync_617 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 2); + float _add_616; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_616) : "f"(p_3[2]), "f"(_shfl_xor_sync_617)); + p_3[2] = _add_616; + float _shfl_xor_sync_619 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 2); + float _add_618; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_618) : "f"(p_3[3]), "f"(_shfl_xor_sync_619)); + p_3[3] = _add_618; + float _shfl_xor_sync_621 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 1); + float _add_620; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_620) : "f"(p_3[0]), "f"(_shfl_xor_sync_621)); + p_3[0] = _add_620; + float _shfl_xor_sync_623 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 1); + float _add_622; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_622) : "f"(p_3[1]), "f"(_shfl_xor_sync_623)); + p_3[1] = _add_622; + float _shfl_xor_sync_625 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 1); + float _add_624; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_624) : "f"(p_3[2]), "f"(_shfl_xor_sync_625)); + p_3[2] = _add_624; + float _shfl_xor_sync_627 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 1); + float _add_626; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_626) : "f"(p_3[3]), "f"(_shfl_xor_sync_627)); + p_3[3] = _add_626; + float _add_628; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_628) : "f"(p_3[0]), "f"(p_3[2])); + float _add_629; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_629) : "f"(p_3[1]), "f"(p_3[3])); float sq_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3) : "f"(_add_596), "f"(_add_597)); - float _shfl_sync_600 = __shfl_sync(0xFFFFFFFF, sq_3, 0); - float _shfl_sync_601 = __shfl_sync(0xFFFFFFFF, sq_3, 8); - float _add_599; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_599) : "f"(_shfl_sync_600), "f"(_shfl_sync_601)); - float _shfl_sync_602 = __shfl_sync(0xFFFFFFFF, sq_3, 16); - float _add_598; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_598) : "f"(_add_599), "f"(_shfl_sync_602)); - float _shfl_sync_603 = __shfl_sync(0xFFFFFFFF, sq_3, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3) : "f"(_add_628), "f"(_add_629)); + float _shfl_sync_632 = __shfl_sync(0xFFFFFFFF, sq_3, 0); + float _shfl_sync_633 = __shfl_sync(0xFFFFFFFF, sq_3, 8); + float _add_631; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_631) : "f"(_shfl_sync_632), "f"(_shfl_sync_633)); + float _shfl_sync_634 = __shfl_sync(0xFFFFFFFF, sq_3, 16); + float _add_630; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_630) : "f"(_add_631), "f"(_shfl_sync_634)); + float _shfl_sync_635 = __shfl_sync(0xFFFFFFFF, sq_3, 24); float dot_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3) : "f"(_add_598), "f"(_shfl_sync_603)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3) : "f"(_add_630), "f"(_shfl_sync_635)); float o_3[4]; - float _fma_605; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_605) : "f"(y_3[0]), "f"(dot_3), "f"(nz_2)); - float _sub_604; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_604) : "f"(gq_3[0]), "f"(_fma_605)); - o_3[0] = mul_f32_rn_noftz(rq_3[0], _sub_604); - float _fma_607; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_607) : "f"(y_3[1]), "f"(dot_3), "f"(nz_2)); - float _sub_606; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_606) : "f"(gq_3[1]), "f"(_fma_607)); - o_3[1] = mul_f32_rn_noftz(rq_3[0], _sub_606); - float _fma_609; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_609) : "f"(y_3[2]), "f"(dot_3), "f"(nz_2)); - float _sub_608; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_608) : "f"(gq_3[2]), "f"(_fma_609)); - o_3[2] = mul_f32_rn_noftz(rq_3[0], _sub_608); - float _fma_611; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_611) : "f"(y_3[3]), "f"(dot_3), "f"(nz_2)); - float _sub_610; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_610) : "f"(gq_3[3]), "f"(_fma_611)); - o_3[3] = mul_f32_rn_noftz(rq_3[0], _sub_610); + float _fma_637; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_637) : "f"(y_3[0]), "f"(dot_3), "f"(nz_2)); + float _sub_636; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_636) : "f"(gq_3[0]), "f"(_fma_637)); + o_3[0] = mul_f32_rn_noftz(rq_3[0], _sub_636); + float _fma_639; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_639) : "f"(y_3[1]), "f"(dot_3), "f"(nz_2)); + float _sub_638; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_638) : "f"(gq_3[1]), "f"(_fma_639)); + o_3[1] = mul_f32_rn_noftz(rq_3[0], _sub_638); + float _fma_641; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_641) : "f"(y_3[2]), "f"(dot_3), "f"(nz_2)); + float _sub_640; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_640) : "f"(gq_3[2]), "f"(_fma_641)); + o_3[2] = mul_f32_rn_noftz(rq_3[0], _sub_640); + float _fma_643; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_643) : "f"(y_3[3]), "f"(dot_3), "f"(nz_2)); + float _sub_642; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_642) : "f"(gq_3[3]), "f"(_fma_643)); + o_3[3] = mul_f32_rn_noftz(rq_3[0], _sub_642); unsigned int ob_3[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -5755,106 +6033,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_3_1[4]; float p_3_1[4]; y_3_1[0] = mul_f32_rn_noftz(xk_3[0], rk_3[0]); - float _fma_612; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_612) : "f"(gk_3[0]), "f"(y_3_1[0]), "f"(nz_2)); - p_3_1[0] = _fma_612; + float _fma_644; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_644) : "f"(gk_3[0]), "f"(y_3_1[0]), "f"(nz_2)); + p_3_1[0] = _fma_644; y_3_1[1] = mul_f32_rn_noftz(xk_3[1], rk_3[0]); - float _fma_613; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_613) : "f"(gk_3[1]), "f"(y_3_1[1]), "f"(nz_2)); - p_3_1[1] = _fma_613; + float _fma_645; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_645) : "f"(gk_3[1]), "f"(y_3_1[1]), "f"(nz_2)); + p_3_1[1] = _fma_645; y_3_1[2] = mul_f32_rn_noftz(xk_3[2], rk_3[0]); - float _fma_614; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_614) : "f"(gk_3[2]), "f"(y_3_1[2]), "f"(nz_2)); - p_3_1[2] = _fma_614; + float _fma_646; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_646) : "f"(gk_3[2]), "f"(y_3_1[2]), "f"(nz_2)); + p_3_1[2] = _fma_646; y_3_1[3] = mul_f32_rn_noftz(xk_3[3], rk_3[0]); - float _fma_615; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_615) : "f"(gk_3[3]), "f"(y_3_1[3]), "f"(nz_2)); - p_3_1[3] = _fma_615; - float _shfl_xor_sync_617 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 4); - float _add_616; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_616) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_617)); - p_3_1[0] = _add_616; - float _shfl_xor_sync_619 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 4); - float _add_618; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_618) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_619)); - p_3_1[1] = _add_618; - float _shfl_xor_sync_621 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 4); - float _add_620; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_620) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_621)); - p_3_1[2] = _add_620; - float _shfl_xor_sync_623 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 4); - float _add_622; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_622) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_623)); - p_3_1[3] = _add_622; - float _shfl_xor_sync_625 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 2); - float _add_624; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_624) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_625)); - p_3_1[0] = _add_624; - float _shfl_xor_sync_627 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 2); - float _add_626; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_626) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_627)); - p_3_1[1] = _add_626; - float _shfl_xor_sync_629 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 2); - float _add_628; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_628) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_629)); - p_3_1[2] = _add_628; - float _shfl_xor_sync_631 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 2); - float _add_630; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_630) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_631)); - p_3_1[3] = _add_630; - float _shfl_xor_sync_633 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 1); - float _add_632; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_632) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_633)); - p_3_1[0] = _add_632; - float _shfl_xor_sync_635 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 1); - float _add_634; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_634) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_635)); - p_3_1[1] = _add_634; - float _shfl_xor_sync_637 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 1); - float _add_636; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_636) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_637)); - p_3_1[2] = _add_636; - float _shfl_xor_sync_639 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 1); - float _add_638; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_638) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_639)); - p_3_1[3] = _add_638; - float _add_640; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_640) : "f"(p_3_1[0]), "f"(p_3_1[2])); - float _add_641; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_641) : "f"(p_3_1[1]), "f"(p_3_1[3])); + float _fma_647; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_647) : "f"(gk_3[3]), "f"(y_3_1[3]), "f"(nz_2)); + p_3_1[3] = _fma_647; + float _shfl_xor_sync_649 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 4); + float _add_648; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_648) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_649)); + p_3_1[0] = _add_648; + float _shfl_xor_sync_651 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 4); + float _add_650; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_650) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_651)); + p_3_1[1] = _add_650; + float _shfl_xor_sync_653 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 4); + float _add_652; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_652) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_653)); + p_3_1[2] = _add_652; + float _shfl_xor_sync_655 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 4); + float _add_654; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_654) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_655)); + p_3_1[3] = _add_654; + float _shfl_xor_sync_657 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 2); + float _add_656; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_656) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_657)); + p_3_1[0] = _add_656; + float _shfl_xor_sync_659 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 2); + float _add_658; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_658) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_659)); + p_3_1[1] = _add_658; + float _shfl_xor_sync_661 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 2); + float _add_660; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_660) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_661)); + p_3_1[2] = _add_660; + float _shfl_xor_sync_663 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 2); + float _add_662; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_662) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_663)); + p_3_1[3] = _add_662; + float _shfl_xor_sync_665 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 1); + float _add_664; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_664) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_665)); + p_3_1[0] = _add_664; + float _shfl_xor_sync_667 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 1); + float _add_666; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_666) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_667)); + p_3_1[1] = _add_666; + float _shfl_xor_sync_669 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 1); + float _add_668; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_668) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_669)); + p_3_1[2] = _add_668; + float _shfl_xor_sync_671 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 1); + float _add_670; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_670) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_671)); + p_3_1[3] = _add_670; + float _add_672; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_672) : "f"(p_3_1[0]), "f"(p_3_1[2])); + float _add_673; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_673) : "f"(p_3_1[1]), "f"(p_3_1[3])); float sq_3_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3_1) : "f"(_add_640), "f"(_add_641)); - float _shfl_sync_644 = __shfl_sync(0xFFFFFFFF, sq_3_1, 0); - float _shfl_sync_645 = __shfl_sync(0xFFFFFFFF, sq_3_1, 8); - float _add_643; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_643) : "f"(_shfl_sync_644), "f"(_shfl_sync_645)); - float _shfl_sync_646 = __shfl_sync(0xFFFFFFFF, sq_3_1, 16); - float _add_642; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_642) : "f"(_add_643), "f"(_shfl_sync_646)); - float _shfl_sync_647 = __shfl_sync(0xFFFFFFFF, sq_3_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3_1) : "f"(_add_672), "f"(_add_673)); + float _shfl_sync_676 = __shfl_sync(0xFFFFFFFF, sq_3_1, 0); + float _shfl_sync_677 = __shfl_sync(0xFFFFFFFF, sq_3_1, 8); + float _add_675; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_675) : "f"(_shfl_sync_676), "f"(_shfl_sync_677)); + float _shfl_sync_678 = __shfl_sync(0xFFFFFFFF, sq_3_1, 16); + float _add_674; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_674) : "f"(_add_675), "f"(_shfl_sync_678)); + float _shfl_sync_679 = __shfl_sync(0xFFFFFFFF, sq_3_1, 24); float dot_3_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3_1) : "f"(_add_642), "f"(_shfl_sync_647)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3_1) : "f"(_add_674), "f"(_shfl_sync_679)); float o_3_1[4]; - float _fma_649; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_649) : "f"(y_3_1[0]), "f"(dot_3_1), "f"(nz_2)); - float _sub_648; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_648) : "f"(gk_3[0]), "f"(_fma_649)); - o_3_1[0] = mul_f32_rn_noftz(rk_3[0], _sub_648); - float _fma_651; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_651) : "f"(y_3_1[1]), "f"(dot_3_1), "f"(nz_2)); - float _sub_650; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_650) : "f"(gk_3[1]), "f"(_fma_651)); - o_3_1[1] = mul_f32_rn_noftz(rk_3[0], _sub_650); - float _fma_653; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_653) : "f"(y_3_1[2]), "f"(dot_3_1), "f"(nz_2)); - float _sub_652; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_652) : "f"(gk_3[2]), "f"(_fma_653)); - o_3_1[2] = mul_f32_rn_noftz(rk_3[0], _sub_652); - float _fma_655; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_655) : "f"(y_3_1[3]), "f"(dot_3_1), "f"(nz_2)); - float _sub_654; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_654) : "f"(gk_3[3]), "f"(_fma_655)); - o_3_1[3] = mul_f32_rn_noftz(rk_3[0], _sub_654); + float _fma_681; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_681) : "f"(y_3_1[0]), "f"(dot_3_1), "f"(nz_2)); + float _sub_680; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_680) : "f"(gk_3[0]), "f"(_fma_681)); + o_3_1[0] = mul_f32_rn_noftz(rk_3[0], _sub_680); + float _fma_683; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_683) : "f"(y_3_1[1]), "f"(dot_3_1), "f"(nz_2)); + float _sub_682; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_682) : "f"(gk_3[1]), "f"(_fma_683)); + o_3_1[1] = mul_f32_rn_noftz(rk_3[0], _sub_682); + float _fma_685; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_685) : "f"(y_3_1[2]), "f"(dot_3_1), "f"(nz_2)); + float _sub_684; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_684) : "f"(gk_3[2]), "f"(_fma_685)); + o_3_1[2] = mul_f32_rn_noftz(rk_3[0], _sub_684); + float _fma_687; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_687) : "f"(y_3_1[3]), "f"(dot_3_1), "f"(nz_2)); + float _sub_686; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_686) : "f"(gk_3[3]), "f"(_fma_687)); + o_3_1[3] = mul_f32_rn_noftz(rk_3[0], _sub_686); unsigned int ob_3_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -5865,8 +6143,8 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM } asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_656 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_656), "f"(tot_1693) : "memory"); + uint32_t _addr_688 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_688), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5895,9 +6173,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_16 = offset_1710 + _tmem_load_29[0]; if (t_49 < clen) { { - auto _gst_657 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_657)); - *(reinterpret_cast(_gst_657) + (0)) = dg_16; + auto _gst_689 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_689)); + *(reinterpret_cast(_gst_689) + (0)) = dg_16; } } int t_50 = 8 * th + 1; @@ -5905,9 +6183,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_17 = offset_1710 + _tmem_load_29[1]; if (t_50 < clen) { { - auto _gst_658 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_658)); - *(reinterpret_cast(_gst_658) + (0)) = dg_17; + auto _gst_690 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_690)); + *(reinterpret_cast(_gst_690) + (0)) = dg_17; } } int t_51 = 8 * th + 2; @@ -5915,9 +6193,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_18 = offset_1710 + _tmem_load_29[2]; if (t_51 < clen) { { - auto _gst_659 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_659)); - *(reinterpret_cast(_gst_659) + (0)) = dg_18; + auto _gst_691 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_691)); + *(reinterpret_cast(_gst_691) + (0)) = dg_18; } } int t_52 = 8 * th + 3; @@ -5925,9 +6203,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_19 = offset_1710 + _tmem_load_29[3]; if (t_52 < clen) { { - auto _gst_660 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_660)); - *(reinterpret_cast(_gst_660) + (0)) = dg_19; + auto _gst_692 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_692)); + *(reinterpret_cast(_gst_692) + (0)) = dg_19; } } int t_53 = 8 * th + 4; @@ -5935,9 +6213,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_20 = offset_1710 + _tmem_load_29[4]; if (t_53 < clen) { { - auto _gst_661 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_661)); - *(reinterpret_cast(_gst_661) + (0)) = dg_20; + auto _gst_693 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_693)); + *(reinterpret_cast(_gst_693) + (0)) = dg_20; } } int t_54 = 8 * th + 5; @@ -5945,9 +6223,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_21 = offset_1710 + _tmem_load_29[5]; if (t_54 < clen) { { - auto _gst_662 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_662)); - *(reinterpret_cast(_gst_662) + (0)) = dg_21; + auto _gst_694 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_694)); + *(reinterpret_cast(_gst_694) + (0)) = dg_21; } } int t_55 = 8 * th + 6; @@ -5955,9 +6233,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_22 = offset_1710 + _tmem_load_29[6]; if (t_55 < clen) { { - auto _gst_663 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_663)); - *(reinterpret_cast(_gst_663) + (0)) = dg_22; + auto _gst_695 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_695)); + *(reinterpret_cast(_gst_695) + (0)) = dg_22; } } int t_56 = 8 * th + 7; @@ -5965,9 +6243,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_23 = offset_1710 + _tmem_load_29[7]; if (t_56 < clen) { { - auto _gst_664 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_664)); - *(reinterpret_cast(_gst_664) + (0)) = dg_23; + auto _gst_696 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_696)); + *(reinterpret_cast(_gst_696) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1708 + t_lo_1709; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu index cff9caa322a..2b8d1329c73 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -199,6 +200,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2267,12 +2284,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_0); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_0 : __fmaf_rn(dl, _rcp_170, e2_0)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2296,12 +2319,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_1); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_1, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_1, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_1 : __fmaf_rn(dl_1, _rcp_174, e2_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2325,12 +2354,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_2); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_2, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_2, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_2 : __fmaf_rn(dl_3, _rcp_178, e2_2)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2354,12 +2389,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_3); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_3, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_3, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_3 : __fmaf_rn(dl_5, _rcp_182, e2_3)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2395,12 +2436,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2420,12 +2467,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2445,12 +2498,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2470,12 +2529,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2523,12 +2588,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2552,12 +2623,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_1 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2581,12 +2658,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2610,12 +2693,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2657,12 +2746,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2686,12 +2781,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2715,12 +2816,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2744,12 +2851,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2820,37 +2933,37 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 d2_56 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_56.x); float e1_12 = approx_exp2(d2_56.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_57.x); float e1_13 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_58.x); float e1_14 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_59.x); float e1_15 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2862,29 +2975,29 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2892,29 +3005,29 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2984,7 +3097,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3016,7 +3134,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3048,7 +3171,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_1)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3080,7 +3208,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_2)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3112,7 +3245,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_3)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3144,7 +3282,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_4)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3176,7 +3319,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_5)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3208,7 +3356,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_6)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3223,31 +3376,31 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_216 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_216; + float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_248; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_217 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_217; + float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_249; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_218 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_218; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_250; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_219 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_219; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_251; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_220 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_220; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_252; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_221 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_221; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_253; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_222 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_222; - float _shfl_xor_sync_223 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_223; - float _shfl_xor_sync_224 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_224; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_254; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_255; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_256; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); if (32 + 8 * th < clen) { @@ -3308,322 +3461,135 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float r_272[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1049 = r_272[0]; - float gc_1050 = gn_1049 - 21.640425f; - int eb_1051 = 98304 + th * -32768; - int ew_1052 = eb_1051 >> 2; - int qbase_1053 = pool_f32_addr + (unsigned int)(eb_1051 + 12288 + dh_lane * 1024); - int sb8_1054 = 48 + 8 * th; - int gb8_1055 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1054 * 128); - int kb8_1056 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1054 * 128); - float dbp_1057[8]; - float _tmem_load_12[8]; - tmem_ld_x8(&_tmem_load_12[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float _tmem_load_13[8]; - tmem_ld_x8(&_tmem_load_13[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float _tmem_load_14[8]; - tmem_ld_x8(&_tmem_load_14[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float _tmem_load_15[8]; - tmem_ld_x8(&_tmem_load_15[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float _tmem_load_16[8]; - tmem_ld_x8(&_tmem_load_16[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_1058[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[0])), "=r"(*reinterpret_cast(&btv_1058[(0) + 1])), "=r"(*reinterpret_cast(&btv_1058[(0) + 2])), "=r"(*reinterpret_cast(&btv_1058[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[4])), "=r"(*reinterpret_cast(&btv_1058[(4) + 1])), "=r"(*reinterpret_cast(&btv_1058[(4) + 2])), "=r"(*reinterpret_cast(&btv_1058[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); + float gn_0 = r_272[0]; + float gc_0 = gn_0 - 21.640425f; + int eb_0 = 98304 + th * -32768; + int ew_0 = eb_0 >> 2; + int qbase_0 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); + int sb8_0 = 48 + 8 * th; + int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); + int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); + int c_qa = 160 + 8 * th; + int c_ka = 176 + 8 * th; + int c_qd = 224 + 8 * th; + int c_kd = 240 + 8 * th; + int c_dk = 48 + 8 * th; + int c_bp = c_ka; + float s_run_0[1]; + s_run_0[0] = 0.0f; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j = 7 - k; + float a_qd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j))); + float a_kd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd + j))); + float a_dk[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk + j))); + float a_qa[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa + j))); + float a_ka[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka + j))); + float btr[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_0 + j)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8_0 + j; + float r_273[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_273[0]; + unsigned int r_274[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_274[0] << 16); + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_275[0] << 16); + float q_m = ((t >= clen) ? 0.0f : q_t); + float dqf = pool_f32[ew_0 + 128 * j + d_lane]; + float dkf = pool_f32[ew_0 + (1024 + 128 * j) + d_lane]; + float dgf = pool_f32[ew_0 + (2048 + 128 * j) + d_lane]; + float bt_t = btr[0]; + float e_dg = approx_exp2(g_t - gc_0); + float e_off = approx_exp2(g_t - gn_0); + float dq2 = __fmaf_rn(a_qa[0], e_off, a_qd[0] * e_dg); + float dk2 = __fmaf_rn(a_ka[0], e_off, a_kd[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + a_dk[0]; + float qe_7 = q_m * e_dg; + float kp_19 = k_t * bt_t; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, a_qd[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg) * a_kd[0]); + float dg_o = __fmaf_rn(q_m, dq2, __fmaf_rn(dk2b - a_dk[0], k_t, cr_7)) + dgf; + s_run_0[0] = s_run_0[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(98304 - 32768 * th >> 2) + (128 * j + d_lane)] = dq_o; + pool_f32[(98304 - 32768 * th >> 2) + (1024 + 128 * j + d_lane)] = dk_o; + } + float tot_0 = s_run_0[0]; + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1059[1]; - s_run_1059[0] = 0.0f; - int t_0 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew_1052 + 896 + d_lane]; - float dkf_0 = pool_f32[ew_1052 + 1920 + d_lane]; - float dgf_0 = pool_f32[ew_1052 + 2944 + d_lane]; - float bt_t_0 = btv_1058[7]; - float e_off_0 = approx_exp2(g_t_0 - gn_1049); - float e_dg_0 = approx_exp2(g_t_0 - gc_1050); - float dq2_0 = __fmaf_rn(_tmem_load_12[7], e_off_0, _tmem_load_14[7] * e_dg_0); - float dk2_0 = __fmaf_rn(_tmem_load_13[7], e_off_0, _tmem_load_15[7] * e_dg_0); - dbp_1057[7] = dk2_0 * k_t_0; - float dk2b_0 = dk2_0 * bt_t_0; - float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; - s_run_1059[0] = s_run_1059[0] + dg_o_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_0; - pool_f32[(98304 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_0; - int t_1 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew_1052 + 768 + d_lane]; - float dkf_1 = pool_f32[ew_1052 + 1792 + d_lane]; - float dgf_1 = pool_f32[ew_1052 + 2816 + d_lane]; - float bt_t_1 = btv_1058[6]; - float e_off_1 = approx_exp2(g_t_1 - gn_1049); - float e_dg_1 = approx_exp2(g_t_1 - gc_1050); - float dq2_1 = __fmaf_rn(_tmem_load_12[6], e_off_1, _tmem_load_14[6] * e_dg_1); - float dk2_1 = __fmaf_rn(_tmem_load_13[6], e_off_1, _tmem_load_15[6] * e_dg_1); - dbp_1057[6] = dk2_1 * k_t_1; - float dk2b_1 = dk2_1 * bt_t_1; - float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + _tmem_load_16[6]; - float dg_o_1 = __fmaf_rn(q_t_1_9, dq2_1, (dk2b_1 - _tmem_load_16[6]) * k_t_1) + dgf_1; - s_run_1059[0] = s_run_1059[0] + dg_o_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_1; - pool_f32[(98304 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_1; - int t_2 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew_1052 + 640 + d_lane]; - float dkf_2 = pool_f32[ew_1052 + 1664 + d_lane]; - float dgf_2 = pool_f32[ew_1052 + 2688 + d_lane]; - float bt_t_2 = btv_1058[5]; - float e_off_2 = approx_exp2(g_t_2 - gn_1049); - float e_dg_2 = approx_exp2(g_t_2 - gc_1050); - float dq2_2 = __fmaf_rn(_tmem_load_12[5], e_off_2, _tmem_load_14[5] * e_dg_2); - float dk2_2 = __fmaf_rn(_tmem_load_13[5], e_off_2, _tmem_load_15[5] * e_dg_2); - dbp_1057[5] = dk2_2 * k_t_2; - float dk2b_2 = dk2_2 * bt_t_2; - float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[5]; - float dg_o_2 = __fmaf_rn(q_t_1_10, dq2_2, (dk2b_2 - _tmem_load_16[5]) * k_t_2) + dgf_2; - s_run_1059[0] = s_run_1059[0] + dg_o_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_2; - pool_f32[(98304 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_2; - int t_3 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew_1052 + 512 + d_lane]; - float dkf_3 = pool_f32[ew_1052 + 1536 + d_lane]; - float dgf_3 = pool_f32[ew_1052 + 2560 + d_lane]; - float bt_t_3 = btv_1058[4]; - float e_off_3 = approx_exp2(g_t_3 - gn_1049); - float e_dg_3 = approx_exp2(g_t_3 - gc_1050); - float dq2_3 = __fmaf_rn(_tmem_load_12[4], e_off_3, _tmem_load_14[4] * e_dg_3); - float dk2_3 = __fmaf_rn(_tmem_load_13[4], e_off_3, _tmem_load_15[4] * e_dg_3); - dbp_1057[4] = dk2_3 * k_t_3; - float dk2b_3 = dk2_3 * bt_t_3; - float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[4]; - float dg_o_3 = __fmaf_rn(q_t_1_11, dq2_3, (dk2b_3 - _tmem_load_16[4]) * k_t_3) + dgf_3; - s_run_1059[0] = s_run_1059[0] + dg_o_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_3; - pool_f32[(98304 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_3; - int t_4 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew_1052 + 384 + d_lane]; - float dkf_4 = pool_f32[ew_1052 + 1408 + d_lane]; - float dgf_4 = pool_f32[ew_1052 + 2432 + d_lane]; - float bt_t_4 = btv_1058[3]; - float e_off_4 = approx_exp2(g_t_4 - gn_1049); - float e_dg_4 = approx_exp2(g_t_4 - gc_1050); - float dq2_4 = __fmaf_rn(_tmem_load_12[3], e_off_4, _tmem_load_14[3] * e_dg_4); - float dk2_4 = __fmaf_rn(_tmem_load_13[3], e_off_4, _tmem_load_15[3] * e_dg_4); - dbp_1057[3] = dk2_4 * k_t_4; - float dk2b_4 = dk2_4 * bt_t_4; - float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[3]; - float dg_o_4 = __fmaf_rn(q_t_1_12, dq2_4, (dk2b_4 - _tmem_load_16[3]) * k_t_4) + dgf_4; - s_run_1059[0] = s_run_1059[0] + dg_o_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_4; - pool_f32[(98304 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_4; - int t_5 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew_1052 + 256 + d_lane]; - float dkf_5 = pool_f32[ew_1052 + 1280 + d_lane]; - float dgf_5 = pool_f32[ew_1052 + 2304 + d_lane]; - float bt_t_5 = btv_1058[2]; - float e_off_5 = approx_exp2(g_t_5 - gn_1049); - float e_dg_5 = approx_exp2(g_t_5 - gc_1050); - float dq2_5 = __fmaf_rn(_tmem_load_12[2], e_off_5, _tmem_load_14[2] * e_dg_5); - float dk2_5 = __fmaf_rn(_tmem_load_13[2], e_off_5, _tmem_load_15[2] * e_dg_5); - dbp_1057[2] = dk2_5 * k_t_5; - float dk2b_5 = dk2_5 * bt_t_5; - float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[2]; - float dg_o_5 = __fmaf_rn(q_t_1_13, dq2_5, (dk2b_5 - _tmem_load_16[2]) * k_t_5) + dgf_5; - s_run_1059[0] = s_run_1059[0] + dg_o_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_5; - pool_f32[(98304 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_5; - int t_6 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew_1052 + 128 + d_lane]; - float dkf_6 = pool_f32[ew_1052 + 1152 + d_lane]; - float dgf_6 = pool_f32[ew_1052 + 2176 + d_lane]; - float bt_t_6 = btv_1058[1]; - float e_off_6 = approx_exp2(g_t_6 - gn_1049); - float e_dg_6 = approx_exp2(g_t_6 - gc_1050); - float dq2_6 = __fmaf_rn(_tmem_load_12[1], e_off_6, _tmem_load_14[1] * e_dg_6); - float dk2_6 = __fmaf_rn(_tmem_load_13[1], e_off_6, _tmem_load_15[1] * e_dg_6); - dbp_1057[1] = dk2_6 * k_t_6; - float dk2b_6 = dk2_6 * bt_t_6; - float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[1]; - float dg_o_6 = __fmaf_rn(q_t_1_14, dq2_6, (dk2b_6 - _tmem_load_16[1]) * k_t_6) + dgf_6; - s_run_1059[0] = s_run_1059[0] + dg_o_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_6; - pool_f32[(98304 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_6; - int t_7 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew_1052 + d_lane]; - float dkf_7 = pool_f32[ew_1052 + 1024 + d_lane]; - float dgf_7 = pool_f32[ew_1052 + 2048 + d_lane]; - float bt_t_7 = btv_1058[0]; - float e_off_7 = approx_exp2(g_t_7 - gn_1049); - float e_dg_7 = approx_exp2(g_t_7 - gc_1050); - float dq2_7 = __fmaf_rn(_tmem_load_12[0], e_off_7, _tmem_load_14[0] * e_dg_7); - float dk2_7 = __fmaf_rn(_tmem_load_13[0], e_off_7, _tmem_load_15[0] * e_dg_7); - dbp_1057[0] = dk2_7 * k_t_7; - float dk2b_7 = dk2_7 * bt_t_7; - float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[0]; - float dg_o_7 = __fmaf_rn(q_t_1_15, dq2_7, (dk2b_7 - _tmem_load_16[0]) * k_t_7) + dgf_7; - s_run_1059[0] = s_run_1059[0] + dg_o_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + d_lane] = dq_o_7; - pool_f32[(98304 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_7; - float tot_1228 = s_run_1059[0]; - int b4_1229 = lane_1 >> 4 & 1; - int b3_1230 = lane_1 >> 3 & 1; - int b2_1231 = lane_1 >> 2 & 1; - float v4_1232[4]; - float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_225 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_225; - float send_1 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_226 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_226; - float send_2 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_227 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_227; - float send_3 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_228 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_228; - float v2_1237[2]; - float send_4 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_229 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_229; - float send_5 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_230 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_230; - float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_231 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_231; - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_232; - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_233; + int b4_0 = lane_1 >> 4 & 1; + int b3_0 = lane_1 >> 3 & 1; + int b2_0 = lane_1 >> 2 & 1; + float v4_0[4]; + float send_0 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float send_1 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float send_2 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); + float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float send_3 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); + float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float v2[2]; + float send2_0 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send2_0, 8); + v2[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float send2_1 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send2_1, 8); + v2[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float send1 = ((b2_0 == 0) ? v2[1] : v2[0]); + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send1, 4); + float v1_0 = ((b2_0 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_263; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_264; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_265; asm volatile("tcgen05.fence::before_thread_sync;"); if (48 + 8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + (48 + 8 * th)) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(98304 - 32768 * th >> 2) + d_lane]; @@ -3723,8 +3689,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; { - uint32_t _addr_234 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_234), "f"(tot_1228) : "memory"); + uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3742,8 +3708,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi = ((th == 1) ? tot_1228 : t_other[0]); - float t_lo = ((th == 1) ? t_other[0] : tot_1228); + float t_hi = ((th == 1) ? tot_0 : t_other[0]); + float t_lo = ((th == 1) ? t_other[0] : tot_0); float offset = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi); float sfx[8]; tmem_ld_x8(&sfx[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); @@ -3756,132 +3722,132 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_235 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_267 = make_float2(gacc[4], gacc[4]); float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_235), "l"(*(const unsigned long long*)&rp_u)); - float _abs_238; - asm("abs.f32 %0, %1;" : "=f"(_abs_238) : "f"(rp_z.x)); - float _neg_237; - asm("neg.f32 %0, %1;" : "=f"(_neg_237) : "f"(_abs_238)); - float _exp_236 = __nv_expf(_neg_237); - float _abs_241; - asm("abs.f32 %0, %1;" : "=f"(_abs_241) : "f"(rp_z.y)); - float _neg_240; - asm("neg.f32 %0, %1;" : "=f"(_neg_240) : "f"(_abs_241)); - float _exp_239 = __nv_expf(_neg_240); - float2 rp_e = make_float2(_exp_236, _exp_239); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&rp_u)); + float _abs_270; + asm("abs.f32 %0, %1;" : "=f"(_abs_270) : "f"(rp_z.x)); + float _neg_269; + asm("neg.f32 %0, %1;" : "=f"(_neg_269) : "f"(_abs_270)); + float _exp_268 = __nv_expf(_neg_269); + float _abs_273; + asm("abs.f32 %0, %1;" : "=f"(_abs_273) : "f"(rp_z.y)); + float _neg_272; + asm("neg.f32 %0, %1;" : "=f"(_neg_272) : "f"(_abs_273)); + float _exp_271 = __nv_expf(_neg_272); + float2 rp_e = make_float2(_exp_268, _exp_271); float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_242 = approx_rcp(rp_den.x); - float _rcp_243 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_244 = make_float2(_rcp_242, _rcp_243); + float _rcp_274 = approx_rcp(rp_den.x); + float _rcp_275 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_276 = make_float2(_rcp_274, _rcp_275); float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_244)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_276)); float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_245 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_277 = make_float2(gacc[5], gacc[5]); float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_245)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_246 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_278 = make_float2(gacc[4], gacc[4]); float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_246), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_249; - asm("abs.f32 %0, %1;" : "=f"(_abs_249) : "f"(rp_z_0.x)); - float _neg_248; - asm("neg.f32 %0, %1;" : "=f"(_neg_248) : "f"(_abs_249)); - float _exp_247 = __nv_expf(_neg_248); - float _abs_252; - asm("abs.f32 %0, %1;" : "=f"(_abs_252) : "f"(rp_z_0.y)); - float _neg_251; - asm("neg.f32 %0, %1;" : "=f"(_neg_251) : "f"(_abs_252)); - float _exp_250 = __nv_expf(_neg_251); - float2 rp_e_0 = make_float2(_exp_247, _exp_250); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_278), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_281; + asm("abs.f32 %0, %1;" : "=f"(_abs_281) : "f"(rp_z_0.x)); + float _neg_280; + asm("neg.f32 %0, %1;" : "=f"(_neg_280) : "f"(_abs_281)); + float _exp_279 = __nv_expf(_neg_280); + float _abs_284; + asm("abs.f32 %0, %1;" : "=f"(_abs_284) : "f"(rp_z_0.y)); + float _neg_283; + asm("neg.f32 %0, %1;" : "=f"(_neg_283) : "f"(_abs_284)); + float _exp_282 = __nv_expf(_neg_283); + float2 rp_e_0 = make_float2(_exp_279, _exp_282); float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_253 = approx_rcp(rp_den_0.x); - float _rcp_254 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_255 = make_float2(_rcp_253, _rcp_254); + float _rcp_285 = approx_rcp(rp_den_0.x); + float _rcp_286 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_287 = make_float2(_rcp_285, _rcp_286); float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_255)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_287)); float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_256 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_288 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_256)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_288)); u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_257 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_289 = make_float2(gacc[4], gacc[4]); float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_257), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_260; - asm("abs.f32 %0, %1;" : "=f"(_abs_260) : "f"(rp_z_1.x)); - float _neg_259; - asm("neg.f32 %0, %1;" : "=f"(_neg_259) : "f"(_abs_260)); - float _exp_258 = __nv_expf(_neg_259); - float _abs_263; - asm("abs.f32 %0, %1;" : "=f"(_abs_263) : "f"(rp_z_1.y)); - float _neg_262; - asm("neg.f32 %0, %1;" : "=f"(_neg_262) : "f"(_abs_263)); - float _exp_261 = __nv_expf(_neg_262); - float2 rp_e_1 = make_float2(_exp_258, _exp_261); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_289), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_292; + asm("abs.f32 %0, %1;" : "=f"(_abs_292) : "f"(rp_z_1.x)); + float _neg_291; + asm("neg.f32 %0, %1;" : "=f"(_neg_291) : "f"(_abs_292)); + float _exp_290 = __nv_expf(_neg_291); + float _abs_295; + asm("abs.f32 %0, %1;" : "=f"(_abs_295) : "f"(rp_z_1.y)); + float _neg_294; + asm("neg.f32 %0, %1;" : "=f"(_neg_294) : "f"(_abs_295)); + float _exp_293 = __nv_expf(_neg_294); + float2 rp_e_1 = make_float2(_exp_290, _exp_293); float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_264 = approx_rcp(rp_den_1.x); - float _rcp_265 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_266 = make_float2(_rcp_264, _rcp_265); + float _rcp_296 = approx_rcp(rp_den_1.x); + float _rcp_297 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_298 = make_float2(_rcp_296, _rcp_297); float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_266)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_298)); float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_267 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_299 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_267)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_299)); u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_268 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_300 = make_float2(gacc[4], gacc[4]); float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_268), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_271; - asm("abs.f32 %0, %1;" : "=f"(_abs_271) : "f"(rp_z_2.x)); - float _neg_270; - asm("neg.f32 %0, %1;" : "=f"(_neg_270) : "f"(_abs_271)); - float _exp_269 = __nv_expf(_neg_270); - float _abs_274; - asm("abs.f32 %0, %1;" : "=f"(_abs_274) : "f"(rp_z_2.y)); - float _neg_273; - asm("neg.f32 %0, %1;" : "=f"(_neg_273) : "f"(_abs_274)); - float _exp_272 = __nv_expf(_neg_273); - float2 rp_e_2 = make_float2(_exp_269, _exp_272); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_303; + asm("abs.f32 %0, %1;" : "=f"(_abs_303) : "f"(rp_z_2.x)); + float _neg_302; + asm("neg.f32 %0, %1;" : "=f"(_neg_302) : "f"(_abs_303)); + float _exp_301 = __nv_expf(_neg_302); + float _abs_306; + asm("abs.f32 %0, %1;" : "=f"(_abs_306) : "f"(rp_z_2.y)); + float _neg_305; + asm("neg.f32 %0, %1;" : "=f"(_neg_305) : "f"(_abs_306)); + float _exp_304 = __nv_expf(_neg_305); + float2 rp_e_2 = make_float2(_exp_301, _exp_304); float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_275 = approx_rcp(rp_den_2.x); - float _rcp_276 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_277 = make_float2(_rcp_275, _rcp_276); + float _rcp_307 = approx_rcp(rp_den_2.x); + float _rcp_308 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_309 = make_float2(_rcp_307, _rcp_308); float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_278 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_310 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_278)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_310)); u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -3896,54 +3862,54 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_279; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_279) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_279; + uint32_t _prmt_311; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_311) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_311; int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_280 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = pq_0[0]; + auto _gst_312 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = pq_0[0]; } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_281; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_281) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_281; + uint32_t _prmt_313; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_313) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_313; int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_282 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = pq_1[0]; + auto _gst_314 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = pq_1[0]; } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_283; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_283) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_283; + uint32_t _prmt_315; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_315; int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_284 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast(_gst_284) + (0)) = pq_2[0]; + auto _gst_316 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = pq_2[0]; } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_285; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_285) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_285; + uint32_t _prmt_317; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_317; int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_286 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast(_gst_286) + (0)) = pq_3[0]; + auto _gst_318 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = pq_3[0]; } } int t_a_0 = 48 + 8 * th; @@ -4008,47 +3974,47 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_287; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_287) : "f"(s_a), "f"(nm_t)); + float _sub_319; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_319) : "f"(s_a), "f"(nm_t)); float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_287), "f"(pa)); - float _sub_288; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_288) : "f"(pa), "f"(nm_t)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_319), "f"(pa)); + float _sub_320; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_320) : "f"(pa), "f"(nm_t)); float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_288), "f"(s_a)); - float _abs_290; - asm("abs.f32 %0, %1;" : "=f"(_abs_290) : "f"(s_a)); - float _abs_291; - asm("abs.f32 %0, %1;" : "=f"(_abs_291) : "f"(pa)); - uint32_t _setp_289_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_289_bits) : "f"(_abs_290), "f"(_abs_291)); - bool _setp_289 = _setp_289_bits != 0; - float nm_err = ((_setp_289) ? nm_es : nm_ex); - float _add_292; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_292) : "f"(c_a), "f"(nm_err)); - c_a = _add_292; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_320), "f"(s_a)); + float _abs_322; + asm("abs.f32 %0, %1;" : "=f"(_abs_322) : "f"(s_a)); + float _abs_323; + asm("abs.f32 %0, %1;" : "=f"(_abs_323) : "f"(pa)); + uint32_t _setp_321_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_321_bits) : "f"(_abs_322), "f"(_abs_323)); + bool _setp_321 = _setp_321_bits != 0; + float nm_err = ((_setp_321) ? nm_es : nm_ex); + float _add_324; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(c_a), "f"(nm_err)); + c_a = _add_324; s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_293; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_293) : "f"(s_b), "f"(nm_t_0)); + float _sub_325; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_325) : "f"(s_b), "f"(nm_t_0)); float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_293), "f"(pb)); - float _sub_294; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_294) : "f"(pb), "f"(nm_t_0)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_325), "f"(pb)); + float _sub_326; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_326) : "f"(pb), "f"(nm_t_0)); float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_294), "f"(s_b)); - float _abs_296; - asm("abs.f32 %0, %1;" : "=f"(_abs_296) : "f"(s_b)); - float _abs_297; - asm("abs.f32 %0, %1;" : "=f"(_abs_297) : "f"(pb)); - uint32_t _setp_295_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_295_bits) : "f"(_abs_296), "f"(_abs_297)); - bool _setp_295 = _setp_295_bits != 0; - float nm_err_0 = ((_setp_295) ? nm_es_0 : nm_ex_0); - float _add_298; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_298) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_298; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_326), "f"(s_b)); + float _abs_328; + asm("abs.f32 %0, %1;" : "=f"(_abs_328) : "f"(s_b)); + float _abs_329; + asm("abs.f32 %0, %1;" : "=f"(_abs_329) : "f"(pb)); + uint32_t _setp_327_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_327_bits) : "f"(_abs_328), "f"(_abs_329)); + bool _setp_327 = _setp_327_bits != 0; + float nm_err_0 = ((_setp_327) ? nm_es_0 : nm_ex_0); + float _add_330; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_330; s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4072,132 +4038,132 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_299 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_331 = make_float2(gacc[4], gacc[4]); float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_299), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_302; - asm("abs.f32 %0, %1;" : "=f"(_abs_302) : "f"(rp_z_3.x)); - float _neg_301; - asm("neg.f32 %0, %1;" : "=f"(_neg_301) : "f"(_abs_302)); - float _exp_300 = __nv_expf(_neg_301); - float _abs_305; - asm("abs.f32 %0, %1;" : "=f"(_abs_305) : "f"(rp_z_3.y)); - float _neg_304; - asm("neg.f32 %0, %1;" : "=f"(_neg_304) : "f"(_abs_305)); - float _exp_303 = __nv_expf(_neg_304); - float2 rp_e_3 = make_float2(_exp_300, _exp_303); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_331), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_334; + asm("abs.f32 %0, %1;" : "=f"(_abs_334) : "f"(rp_z_3.x)); + float _neg_333; + asm("neg.f32 %0, %1;" : "=f"(_neg_333) : "f"(_abs_334)); + float _exp_332 = __nv_expf(_neg_333); + float _abs_337; + asm("abs.f32 %0, %1;" : "=f"(_abs_337) : "f"(rp_z_3.y)); + float _neg_336; + asm("neg.f32 %0, %1;" : "=f"(_neg_336) : "f"(_abs_337)); + float _exp_335 = __nv_expf(_neg_336); + float2 rp_e_3 = make_float2(_exp_332, _exp_335); float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_306 = approx_rcp(rp_den_3.x); - float _rcp_307 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_308 = make_float2(_rcp_306, _rcp_307); + float _rcp_338 = approx_rcp(rp_den_3.x); + float _rcp_339 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_340 = make_float2(_rcp_338, _rcp_339); float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_308)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_340)); float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_309 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_341 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_310 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_342 = make_float2(gacc[4], gacc[4]); float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_310), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_313; - asm("abs.f32 %0, %1;" : "=f"(_abs_313) : "f"(rp_z_4.x)); - float _neg_312; - asm("neg.f32 %0, %1;" : "=f"(_neg_312) : "f"(_abs_313)); - float _exp_311 = __nv_expf(_neg_312); - float _abs_316; - asm("abs.f32 %0, %1;" : "=f"(_abs_316) : "f"(rp_z_4.y)); - float _neg_315; - asm("neg.f32 %0, %1;" : "=f"(_neg_315) : "f"(_abs_316)); - float _exp_314 = __nv_expf(_neg_315); - float2 rp_e_4 = make_float2(_exp_311, _exp_314); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_342), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_345; + asm("abs.f32 %0, %1;" : "=f"(_abs_345) : "f"(rp_z_4.x)); + float _neg_344; + asm("neg.f32 %0, %1;" : "=f"(_neg_344) : "f"(_abs_345)); + float _exp_343 = __nv_expf(_neg_344); + float _abs_348; + asm("abs.f32 %0, %1;" : "=f"(_abs_348) : "f"(rp_z_4.y)); + float _neg_347; + asm("neg.f32 %0, %1;" : "=f"(_neg_347) : "f"(_abs_348)); + float _exp_346 = __nv_expf(_neg_347); + float2 rp_e_4 = make_float2(_exp_343, _exp_346); float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_317 = approx_rcp(rp_den_4.x); - float _rcp_318 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_319 = make_float2(_rcp_317, _rcp_318); + float _rcp_349 = approx_rcp(rp_den_4.x); + float _rcp_350 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_351 = make_float2(_rcp_349, _rcp_350); float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_319)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_351)); float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_320 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_352 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_320)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_352)); u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_321 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_353 = make_float2(gacc[4], gacc[4]); float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_321), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_324; - asm("abs.f32 %0, %1;" : "=f"(_abs_324) : "f"(rp_z_5.x)); - float _neg_323; - asm("neg.f32 %0, %1;" : "=f"(_neg_323) : "f"(_abs_324)); - float _exp_322 = __nv_expf(_neg_323); - float _abs_327; - asm("abs.f32 %0, %1;" : "=f"(_abs_327) : "f"(rp_z_5.y)); - float _neg_326; - asm("neg.f32 %0, %1;" : "=f"(_neg_326) : "f"(_abs_327)); - float _exp_325 = __nv_expf(_neg_326); - float2 rp_e_5 = make_float2(_exp_322, _exp_325); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_356; + asm("abs.f32 %0, %1;" : "=f"(_abs_356) : "f"(rp_z_5.x)); + float _neg_355; + asm("neg.f32 %0, %1;" : "=f"(_neg_355) : "f"(_abs_356)); + float _exp_354 = __nv_expf(_neg_355); + float _abs_359; + asm("abs.f32 %0, %1;" : "=f"(_abs_359) : "f"(rp_z_5.y)); + float _neg_358; + asm("neg.f32 %0, %1;" : "=f"(_neg_358) : "f"(_abs_359)); + float _exp_357 = __nv_expf(_neg_358); + float2 rp_e_5 = make_float2(_exp_354, _exp_357); float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_328 = approx_rcp(rp_den_5.x); - float _rcp_329 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_330 = make_float2(_rcp_328, _rcp_329); + float _rcp_360 = approx_rcp(rp_den_5.x); + float _rcp_361 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_362 = make_float2(_rcp_360, _rcp_361); float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_330)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_362)); float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_331 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_363 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_331)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_363)); u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_332 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_364 = make_float2(gacc[4], gacc[4]); float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_332), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_335; - asm("abs.f32 %0, %1;" : "=f"(_abs_335) : "f"(rp_z_6.x)); - float _neg_334; - asm("neg.f32 %0, %1;" : "=f"(_neg_334) : "f"(_abs_335)); - float _exp_333 = __nv_expf(_neg_334); - float _abs_338; - asm("abs.f32 %0, %1;" : "=f"(_abs_338) : "f"(rp_z_6.y)); - float _neg_337; - asm("neg.f32 %0, %1;" : "=f"(_neg_337) : "f"(_abs_338)); - float _exp_336 = __nv_expf(_neg_337); - float2 rp_e_6 = make_float2(_exp_333, _exp_336); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_364), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_367; + asm("abs.f32 %0, %1;" : "=f"(_abs_367) : "f"(rp_z_6.x)); + float _neg_366; + asm("neg.f32 %0, %1;" : "=f"(_neg_366) : "f"(_abs_367)); + float _exp_365 = __nv_expf(_neg_366); + float _abs_370; + asm("abs.f32 %0, %1;" : "=f"(_abs_370) : "f"(rp_z_6.y)); + float _neg_369; + asm("neg.f32 %0, %1;" : "=f"(_neg_369) : "f"(_abs_370)); + float _exp_368 = __nv_expf(_neg_369); + float2 rp_e_6 = make_float2(_exp_365, _exp_368); float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_339 = approx_rcp(rp_den_6.x); - float _rcp_340 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_341 = make_float2(_rcp_339, _rcp_340); + float _rcp_371 = approx_rcp(rp_den_6.x); + float _rcp_372 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_373 = make_float2(_rcp_371, _rcp_372); float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_373)); float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_342 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_374 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_342)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_374)); u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4212,54 +4178,54 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_343; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_343) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_343; + uint32_t _prmt_375; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_375) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_375; int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_344 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_344)); - *(reinterpret_cast(_gst_344) + (0)) = pq_4[0]; + auto _gst_376 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_376)); + *(reinterpret_cast(_gst_376) + (0)) = pq_4[0]; } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_345; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_345) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_345; + uint32_t _prmt_377; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_377) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_377; int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_346 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = pq_5[0]; + auto _gst_378 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = pq_5[0]; } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_347; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_347) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_347; + uint32_t _prmt_379; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_379; int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_348 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = pq_6[0]; + auto _gst_380 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = pq_6[0]; } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_349; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_349) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_349; + uint32_t _prmt_381; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_381; int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_350 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = pq_7[0]; + auto _gst_382 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = pq_7[0]; } } int t_a_8 = 32 + 8 * th; @@ -4324,47 +4290,47 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_351; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_351) : "f"(s_a_0), "f"(nm_t_1)); + float _sub_383; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_383) : "f"(s_a_0), "f"(nm_t_1)); float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_351), "f"(pa_0)); - float _sub_352; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_352) : "f"(pa_0), "f"(nm_t_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_383), "f"(pa_0)); + float _sub_384; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_384) : "f"(pa_0), "f"(nm_t_1)); float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_352), "f"(s_a_0)); - float _abs_354; - asm("abs.f32 %0, %1;" : "=f"(_abs_354) : "f"(s_a_0)); - float _abs_355; - asm("abs.f32 %0, %1;" : "=f"(_abs_355) : "f"(pa_0)); - uint32_t _setp_353_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_353_bits) : "f"(_abs_354), "f"(_abs_355)); - bool _setp_353 = _setp_353_bits != 0; - float nm_err_1 = ((_setp_353) ? nm_es_1 : nm_ex_1); - float _add_356; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_356) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_356; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_384), "f"(s_a_0)); + float _abs_386; + asm("abs.f32 %0, %1;" : "=f"(_abs_386) : "f"(s_a_0)); + float _abs_387; + asm("abs.f32 %0, %1;" : "=f"(_abs_387) : "f"(pa_0)); + uint32_t _setp_385_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_385_bits) : "f"(_abs_386), "f"(_abs_387)); + bool _setp_385 = _setp_385_bits != 0; + float nm_err_1 = ((_setp_385) ? nm_es_1 : nm_ex_1); + float _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_388; s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_357; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_357) : "f"(s_b_0), "f"(nm_t_2)); + float _sub_389; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_389) : "f"(s_b_0), "f"(nm_t_2)); float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_357), "f"(pb_0)); - float _sub_358; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_358) : "f"(pb_0), "f"(nm_t_2)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_389), "f"(pb_0)); + float _sub_390; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_390) : "f"(pb_0), "f"(nm_t_2)); float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_358), "f"(s_b_0)); - float _abs_360; - asm("abs.f32 %0, %1;" : "=f"(_abs_360) : "f"(s_b_0)); - float _abs_361; - asm("abs.f32 %0, %1;" : "=f"(_abs_361) : "f"(pb_0)); - uint32_t _setp_359_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_359_bits) : "f"(_abs_360), "f"(_abs_361)); - bool _setp_359 = _setp_359_bits != 0; - float nm_err_2 = ((_setp_359) ? nm_es_2 : nm_ex_2); - float _add_362; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_362) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_362; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_390), "f"(s_b_0)); + float _abs_392; + asm("abs.f32 %0, %1;" : "=f"(_abs_392) : "f"(s_b_0)); + float _abs_393; + asm("abs.f32 %0, %1;" : "=f"(_abs_393) : "f"(pb_0)); + uint32_t _setp_391_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_391_bits) : "f"(_abs_392), "f"(_abs_393)); + bool _setp_391 = _setp_391_bits != 0; + float nm_err_2 = ((_setp_391) ? nm_es_2 : nm_ex_2); + float _add_394; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_394; s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4372,8 +4338,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_363 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_363), "f"(off_acc[0]) : "memory"); + uint32_t _addr_395 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_395), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4399,324 +4365,137 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1292 = r_297[0]; - float gc_1293 = gn_1292 - 21.640425f; - int eb_1294 = 81920 + th * -32768; - int ew_1295 = eb_1294 >> 2; - int qbase_1296 = pool_f32_addr + (unsigned int)(eb_1294 + 12288 + dh_lane * 1024); - int sb8_1297 = 16 + 8 * th; - int gb8_1298 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1297 * 128); - int kb8_1299 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1297 * 128); - float dbp_1300[8]; - float _tmem_load_20[8]; - tmem_ld_x8(&_tmem_load_20[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); - float _tmem_load_21[8]; - tmem_ld_x8(&_tmem_load_21[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); - float _tmem_load_22[8]; - tmem_ld_x8(&_tmem_load_22[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); - float _tmem_load_23[8]; - tmem_ld_x8(&_tmem_load_23[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); - float _tmem_load_24[8]; - tmem_ld_x8(&_tmem_load_24[0], taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th)); - float btv_1301[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1301[0])), "=r"(*reinterpret_cast(&btv_1301[(0) + 1])), "=r"(*reinterpret_cast(&btv_1301[(0) + 2])), "=r"(*reinterpret_cast(&btv_1301[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1301[4])), "=r"(*reinterpret_cast(&btv_1301[(4) + 1])), "=r"(*reinterpret_cast(&btv_1301[(4) + 2])), "=r"(*reinterpret_cast(&btv_1301[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th) + 16))); + float r_276[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1 = r_276[0]; + float gc_1 = gn_1 - 21.640425f; + int eb_1 = 81920 + th * -32768; + int ew_1 = eb_1 >> 2; + int qbase_1 = pool_f32_addr + (unsigned int)(eb_1 + 12288 + dh_lane * 1024); + int sb8_1 = 16 + 8 * th; + int gb8_1 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1 * 128); + int kb8_1 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1 * 128); + int c_qa_0 = 64 + 8 * th; + int c_ka_0 = 80 + 8 * th; + int c_qd_0 = 96 + 8 * th; + int c_kd_0 = 112 + 8 * th; + int c_dk_0 = 16 + 8 * th; + int c_bp_0 = c_ka_0; + float s_run_1[1]; + s_run_1[0] = 0.0f; + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_1 = 7 - k_1; + float a_qd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1))); + float a_kd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_0 + j_1))); + float a_dk_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_0 + j_1))); + float a_qa_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa_0 + j_1))); + float a_ka_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka_0 + j_1))); + float btr_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_1 + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_1 + j_1; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1 = __uint_as_float(r_279[0] << 16); + float q_m_1 = ((t_1 >= clen) ? 0.0f : q_t_1); + float dqf_1 = pool_f32[ew_1 + 128 * j_1 + d_lane]; + float dkf_1 = pool_f32[ew_1 + (1024 + 128 * j_1) + d_lane]; + float dgf_1 = pool_f32[ew_1 + (2048 + 128 * j_1) + d_lane]; + float bt_t_1 = btr_1[0]; + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float e_off_1 = approx_exp2(g_t_1 - gn_1); + float dq2_1 = __fmaf_rn(a_qa_1[0], e_off_1, a_qd_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(a_ka_1[0], e_off_1, a_kd_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + a_dk_1[0]; + float qe_8 = q_m_1 * e_dg_1; + float kp_20 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_1; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, a_qd_1[0], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_1) * a_kd_1[0]); + float dg_o_1 = __fmaf_rn(q_m_1, dq2_1, __fmaf_rn(dk2b_1 - a_dk_1[0], k_t_1, cr_8)) + dgf_1; + s_run_1[0] = s_run_1[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1)), "f"(s_run_1[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_1)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(81920 - 32768 * th >> 2) + (128 * j_1 + d_lane)] = dq_o_1; + pool_f32[(81920 - 32768 * th >> 2) + (1024 + 128 * j_1 + d_lane)] = dk_o_1; + } + float tot_1 = s_run_1[0]; + float dbp_1[8]; + tmem_ld_x8(&dbp_1[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1302[1]; - s_run_1302[0] = 0.0f; - int t_8 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_1295 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_1295 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_1295 + 2944 + d_lane]; - float bt_t_8 = btv_1301[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_1292); - float e_dg_8 = approx_exp2(g_t_8 - gc_1293); - float dq2_8 = __fmaf_rn(_tmem_load_20[7], e_off_8, _tmem_load_22[7] * e_dg_8); - float dk2_8 = __fmaf_rn(_tmem_load_21[7], e_off_8, _tmem_load_23[7] * e_dg_8); - dbp_1300[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_24[7]; - float dg_o_8 = __fmaf_rn(q_t_1_16, dq2_8, (dk2b_8 - _tmem_load_24[7]) * k_t_8) + dgf_8; - s_run_1302[0] = s_run_1302[0] + dg_o_8; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_8; - pool_f32[(81920 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_8; - int t_9 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_1295 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_1295 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_1295 + 2816 + d_lane]; - float bt_t_9 = btv_1301[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_1292); - float e_dg_9 = approx_exp2(g_t_9 - gc_1293); - float dq2_9 = __fmaf_rn(_tmem_load_20[6], e_off_9, _tmem_load_22[6] * e_dg_9); - float dk2_9 = __fmaf_rn(_tmem_load_21[6], e_off_9, _tmem_load_23[6] * e_dg_9); - dbp_1300[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[6]; - float dg_o_9 = __fmaf_rn(q_t_1_17, dq2_9, (dk2b_9 - _tmem_load_24[6]) * k_t_9) + dgf_9; - s_run_1302[0] = s_run_1302[0] + dg_o_9; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_9; - pool_f32[(81920 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_9; - int t_10 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_1295 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_1295 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_1295 + 2688 + d_lane]; - float bt_t_10 = btv_1301[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_1292); - float e_dg_10 = approx_exp2(g_t_10 - gc_1293); - float dq2_10 = __fmaf_rn(_tmem_load_20[5], e_off_10, _tmem_load_22[5] * e_dg_10); - float dk2_10 = __fmaf_rn(_tmem_load_21[5], e_off_10, _tmem_load_23[5] * e_dg_10); - dbp_1300[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[5]; - float dg_o_10 = __fmaf_rn(q_t_1_18, dq2_10, (dk2b_10 - _tmem_load_24[5]) * k_t_10) + dgf_10; - s_run_1302[0] = s_run_1302[0] + dg_o_10; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_10; - pool_f32[(81920 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_10; - int t_11 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_1295 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_1295 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_1295 + 2560 + d_lane]; - float bt_t_11 = btv_1301[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_1292); - float e_dg_11 = approx_exp2(g_t_11 - gc_1293); - float dq2_11 = __fmaf_rn(_tmem_load_20[4], e_off_11, _tmem_load_22[4] * e_dg_11); - float dk2_11 = __fmaf_rn(_tmem_load_21[4], e_off_11, _tmem_load_23[4] * e_dg_11); - dbp_1300[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[4]; - float dg_o_11 = __fmaf_rn(q_t_1_19, dq2_11, (dk2b_11 - _tmem_load_24[4]) * k_t_11) + dgf_11; - s_run_1302[0] = s_run_1302[0] + dg_o_11; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_11; - pool_f32[(81920 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_11; - int t_12 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_1295 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_1295 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_1295 + 2432 + d_lane]; - float bt_t_12 = btv_1301[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_1292); - float e_dg_12 = approx_exp2(g_t_12 - gc_1293); - float dq2_12 = __fmaf_rn(_tmem_load_20[3], e_off_12, _tmem_load_22[3] * e_dg_12); - float dk2_12 = __fmaf_rn(_tmem_load_21[3], e_off_12, _tmem_load_23[3] * e_dg_12); - dbp_1300[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[3]; - float dg_o_12 = __fmaf_rn(q_t_1_20, dq2_12, (dk2b_12 - _tmem_load_24[3]) * k_t_12) + dgf_12; - s_run_1302[0] = s_run_1302[0] + dg_o_12; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_12; - pool_f32[(81920 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_12; - int t_13 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_1295 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_1295 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_1295 + 2304 + d_lane]; - float bt_t_13 = btv_1301[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_1292); - float e_dg_13 = approx_exp2(g_t_13 - gc_1293); - float dq2_13 = __fmaf_rn(_tmem_load_20[2], e_off_13, _tmem_load_22[2] * e_dg_13); - float dk2_13 = __fmaf_rn(_tmem_load_21[2], e_off_13, _tmem_load_23[2] * e_dg_13); - dbp_1300[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[2]; - float dg_o_13 = __fmaf_rn(q_t_1_21, dq2_13, (dk2b_13 - _tmem_load_24[2]) * k_t_13) + dgf_13; - s_run_1302[0] = s_run_1302[0] + dg_o_13; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_13; - pool_f32[(81920 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_13; - int t_14 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_1295 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_1295 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_1295 + 2176 + d_lane]; - float bt_t_14 = btv_1301[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_1292); - float e_dg_14 = approx_exp2(g_t_14 - gc_1293); - float dq2_14 = __fmaf_rn(_tmem_load_20[1], e_off_14, _tmem_load_22[1] * e_dg_14); - float dk2_14 = __fmaf_rn(_tmem_load_21[1], e_off_14, _tmem_load_23[1] * e_dg_14); - dbp_1300[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[1]; - float dg_o_14 = __fmaf_rn(q_t_1_22, dq2_14, (dk2b_14 - _tmem_load_24[1]) * k_t_14) + dgf_14; - s_run_1302[0] = s_run_1302[0] + dg_o_14; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_14; - pool_f32[(81920 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_14; - int t_15 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_1295 + d_lane]; - float dkf_15 = pool_f32[ew_1295 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_1295 + 2048 + d_lane]; - float bt_t_15 = btv_1301[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_1292); - float e_dg_15 = approx_exp2(g_t_15 - gc_1293); - float dq2_15 = __fmaf_rn(_tmem_load_20[0], e_off_15, _tmem_load_22[0] * e_dg_15); - float dk2_15 = __fmaf_rn(_tmem_load_21[0], e_off_15, _tmem_load_23[0] * e_dg_15); - dbp_1300[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[0]; - float dg_o_15 = __fmaf_rn(q_t_1_23, dq2_15, (dk2b_15 - _tmem_load_24[0]) * k_t_15) + dgf_15; - s_run_1302[0] = s_run_1302[0] + dg_o_15; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + d_lane] = dq_o_15; - pool_f32[(81920 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_15; - float tot_1471 = s_run_1302[0]; - int b4_1472 = lane_1 >> 4 & 1; - int b3_1473 = lane_1 >> 3 & 1; - int b2_1474 = lane_1 >> 2 & 1; - float v4_1475[4]; - float send_6 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_364 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_364; - float send_7 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_365 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_365; - float send_8 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_366 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_366; - float send_9 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_367; - float v2_1480[2]; - float send_10 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_368; - float send_11 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_369; - float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_370; - float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_371; - float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_372; + int b4_1 = lane_1 >> 4 & 1; + int b3_1 = lane_1 >> 3 & 1; + int b2_1 = lane_1 >> 2 & 1; + float v4_1[4]; + float send_4 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); + float _shfl_xor_sync_396 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_396; + float send_5 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); + float _shfl_xor_sync_397 = __shfl_xor_sync(0xFFFFFFFF, send_5, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_397; + float send_6 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); + float _shfl_xor_sync_398 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_398; + float send_7 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); + float _shfl_xor_sync_399 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_399; + float v2_16[2]; + float send2_2 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); + float _shfl_xor_sync_400 = __shfl_xor_sync(0xFFFFFFFF, send2_2, 8); + v2_16[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_400; + float send2_3 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); + float _shfl_xor_sync_401 = __shfl_xor_sync(0xFFFFFFFF, send2_3, 8); + v2_16[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_401; + float send1_0 = ((b2_1 == 0) ? v2_16[1] : v2_16[0]); + float _shfl_xor_sync_402 = __shfl_xor_sync(0xFFFFFFFF, send1_0, 4); + float v1_1 = ((b2_1 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_402; + float _shfl_xor_sync_403 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_403; + float _shfl_xor_sync_404 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_404; asm volatile("tcgen05.fence::before_thread_sync;"); if (16 + 8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + (16 + 8 * th)) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(81920 - 32768 * th >> 2) + d_lane]; @@ -4771,8 +4550,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_373 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_373), "f"(tot_1471) : "memory"); + uint32_t _addr_405 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_405), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4790,8 +4569,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other_0[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other_0[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi_1 = ((th == 1) ? tot_1471 : t_other_0[0]); - float t_lo_1 = ((th == 1) ? t_other_0[0] : tot_1471); + float t_hi_1 = ((th == 1) ? tot_1 : t_other_0[0]); + float t_lo_1 = ((th == 1) ? t_other_0[0] : tot_1); float offset_1 = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi_1); float sfx_1[8]; tmem_ld_x8(&sfx_1[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); @@ -4804,132 +4583,132 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_374 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_406 = make_float2(gacc[4], gacc[4]); float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_374), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_377; - asm("abs.f32 %0, %1;" : "=f"(_abs_377) : "f"(rp_z_7.x)); - float _neg_376; - asm("neg.f32 %0, %1;" : "=f"(_neg_376) : "f"(_abs_377)); - float _exp_375 = __nv_expf(_neg_376); - float _abs_380; - asm("abs.f32 %0, %1;" : "=f"(_abs_380) : "f"(rp_z_7.y)); - float _neg_379; - asm("neg.f32 %0, %1;" : "=f"(_neg_379) : "f"(_abs_380)); - float _exp_378 = __nv_expf(_neg_379); - float2 rp_e_7 = make_float2(_exp_375, _exp_378); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_406), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_409; + asm("abs.f32 %0, %1;" : "=f"(_abs_409) : "f"(rp_z_7.x)); + float _neg_408; + asm("neg.f32 %0, %1;" : "=f"(_neg_408) : "f"(_abs_409)); + float _exp_407 = __nv_expf(_neg_408); + float _abs_412; + asm("abs.f32 %0, %1;" : "=f"(_abs_412) : "f"(rp_z_7.y)); + float _neg_411; + asm("neg.f32 %0, %1;" : "=f"(_neg_411) : "f"(_abs_412)); + float _exp_410 = __nv_expf(_neg_411); + float2 rp_e_7 = make_float2(_exp_407, _exp_410); float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_381 = approx_rcp(rp_den_7.x); - float _rcp_382 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_383 = make_float2(_rcp_381, _rcp_382); + float _rcp_413 = approx_rcp(rp_den_7.x); + float _rcp_414 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_415 = make_float2(_rcp_413, _rcp_414); float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_383)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_415)); float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_384 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_416 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_384)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_416)); u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_385 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_417 = make_float2(gacc[4], gacc[4]); float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_385), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_388; - asm("abs.f32 %0, %1;" : "=f"(_abs_388) : "f"(rp_z_8.x)); - float _neg_387; - asm("neg.f32 %0, %1;" : "=f"(_neg_387) : "f"(_abs_388)); - float _exp_386 = __nv_expf(_neg_387); - float _abs_391; - asm("abs.f32 %0, %1;" : "=f"(_abs_391) : "f"(rp_z_8.y)); - float _neg_390; - asm("neg.f32 %0, %1;" : "=f"(_neg_390) : "f"(_abs_391)); - float _exp_389 = __nv_expf(_neg_390); - float2 rp_e_8 = make_float2(_exp_386, _exp_389); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_417), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_420; + asm("abs.f32 %0, %1;" : "=f"(_abs_420) : "f"(rp_z_8.x)); + float _neg_419; + asm("neg.f32 %0, %1;" : "=f"(_neg_419) : "f"(_abs_420)); + float _exp_418 = __nv_expf(_neg_419); + float _abs_423; + asm("abs.f32 %0, %1;" : "=f"(_abs_423) : "f"(rp_z_8.y)); + float _neg_422; + asm("neg.f32 %0, %1;" : "=f"(_neg_422) : "f"(_abs_423)); + float _exp_421 = __nv_expf(_neg_422); + float2 rp_e_8 = make_float2(_exp_418, _exp_421); float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_392 = approx_rcp(rp_den_8.x); - float _rcp_393 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_394 = make_float2(_rcp_392, _rcp_393); + float _rcp_424 = approx_rcp(rp_den_8.x); + float _rcp_425 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_426 = make_float2(_rcp_424, _rcp_425); float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_394)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_426)); float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_395 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_427 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_395)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_427)); u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_396 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_428 = make_float2(gacc[4], gacc[4]); float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_396), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_399; - asm("abs.f32 %0, %1;" : "=f"(_abs_399) : "f"(rp_z_9.x)); - float _neg_398; - asm("neg.f32 %0, %1;" : "=f"(_neg_398) : "f"(_abs_399)); - float _exp_397 = __nv_expf(_neg_398); - float _abs_402; - asm("abs.f32 %0, %1;" : "=f"(_abs_402) : "f"(rp_z_9.y)); - float _neg_401; - asm("neg.f32 %0, %1;" : "=f"(_neg_401) : "f"(_abs_402)); - float _exp_400 = __nv_expf(_neg_401); - float2 rp_e_9 = make_float2(_exp_397, _exp_400); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_428), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_431; + asm("abs.f32 %0, %1;" : "=f"(_abs_431) : "f"(rp_z_9.x)); + float _neg_430; + asm("neg.f32 %0, %1;" : "=f"(_neg_430) : "f"(_abs_431)); + float _exp_429 = __nv_expf(_neg_430); + float _abs_434; + asm("abs.f32 %0, %1;" : "=f"(_abs_434) : "f"(rp_z_9.y)); + float _neg_433; + asm("neg.f32 %0, %1;" : "=f"(_neg_433) : "f"(_abs_434)); + float _exp_432 = __nv_expf(_neg_433); + float2 rp_e_9 = make_float2(_exp_429, _exp_432); float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_403 = approx_rcp(rp_den_9.x); - float _rcp_404 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_405 = make_float2(_rcp_403, _rcp_404); + float _rcp_435 = approx_rcp(rp_den_9.x); + float _rcp_436 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_437 = make_float2(_rcp_435, _rcp_436); float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_405)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_437)); float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_406 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_438 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_406)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_438)); u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_407 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_439 = make_float2(gacc[4], gacc[4]); float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_407), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_410; - asm("abs.f32 %0, %1;" : "=f"(_abs_410) : "f"(rp_z_10.x)); - float _neg_409; - asm("neg.f32 %0, %1;" : "=f"(_neg_409) : "f"(_abs_410)); - float _exp_408 = __nv_expf(_neg_409); - float _abs_413; - asm("abs.f32 %0, %1;" : "=f"(_abs_413) : "f"(rp_z_10.y)); - float _neg_412; - asm("neg.f32 %0, %1;" : "=f"(_neg_412) : "f"(_abs_413)); - float _exp_411 = __nv_expf(_neg_412); - float2 rp_e_10 = make_float2(_exp_408, _exp_411); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_439), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_442; + asm("abs.f32 %0, %1;" : "=f"(_abs_442) : "f"(rp_z_10.x)); + float _neg_441; + asm("neg.f32 %0, %1;" : "=f"(_neg_441) : "f"(_abs_442)); + float _exp_440 = __nv_expf(_neg_441); + float _abs_445; + asm("abs.f32 %0, %1;" : "=f"(_abs_445) : "f"(rp_z_10.y)); + float _neg_444; + asm("neg.f32 %0, %1;" : "=f"(_neg_444) : "f"(_abs_445)); + float _exp_443 = __nv_expf(_neg_444); + float2 rp_e_10 = make_float2(_exp_440, _exp_443); float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_414 = approx_rcp(rp_den_10.x); - float _rcp_415 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_416 = make_float2(_rcp_414, _rcp_415); + float _rcp_446 = approx_rcp(rp_den_10.x); + float _rcp_447 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_448 = make_float2(_rcp_446, _rcp_447); float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_416)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_448)); float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_417 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_449 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_417)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_449)); u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -4944,54 +4723,54 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_418; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_418) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_418; + uint32_t _prmt_450; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_450) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_450; int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_419 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_419)); - *(reinterpret_cast(_gst_419) + (0)) = pq_8[0]; + auto _gst_451 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_451)); + *(reinterpret_cast(_gst_451) + (0)) = pq_8[0]; } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_420; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_420) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_420; + uint32_t _prmt_452; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_452) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_452; int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_421 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_421)); - *(reinterpret_cast(_gst_421) + (0)) = pq_9[0]; + auto _gst_453 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_453)); + *(reinterpret_cast(_gst_453) + (0)) = pq_9[0]; } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_422; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_422) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_422; + uint32_t _prmt_454; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_454) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_454; int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_423 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_423)); - *(reinterpret_cast(_gst_423) + (0)) = pq_10[0]; + auto _gst_455 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_455)); + *(reinterpret_cast(_gst_455) + (0)) = pq_10[0]; } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_424; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_424) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_424; + uint32_t _prmt_456; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_456) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_456; int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_425 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_425)); - *(reinterpret_cast(_gst_425) + (0)) = pq_11[0]; + auto _gst_457 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_457)); + *(reinterpret_cast(_gst_457) + (0)) = pq_11[0]; } } int t_a_16 = 16 + 8 * th; @@ -5056,47 +4835,47 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_426; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_426) : "f"(s_a_1), "f"(nm_t_3)); + float _sub_458; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_458) : "f"(s_a_1), "f"(nm_t_3)); float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_426), "f"(pa_1)); - float _sub_427; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_427) : "f"(pa_1), "f"(nm_t_3)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_458), "f"(pa_1)); + float _sub_459; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_459) : "f"(pa_1), "f"(nm_t_3)); float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_427), "f"(s_a_1)); - float _abs_429; - asm("abs.f32 %0, %1;" : "=f"(_abs_429) : "f"(s_a_1)); - float _abs_430; - asm("abs.f32 %0, %1;" : "=f"(_abs_430) : "f"(pa_1)); - uint32_t _setp_428_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_428_bits) : "f"(_abs_429), "f"(_abs_430)); - bool _setp_428 = _setp_428_bits != 0; - float nm_err_3 = ((_setp_428) ? nm_es_3 : nm_ex_3); - float _add_431; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_431) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_431; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_459), "f"(s_a_1)); + float _abs_461; + asm("abs.f32 %0, %1;" : "=f"(_abs_461) : "f"(s_a_1)); + float _abs_462; + asm("abs.f32 %0, %1;" : "=f"(_abs_462) : "f"(pa_1)); + uint32_t _setp_460_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_460_bits) : "f"(_abs_461), "f"(_abs_462)); + bool _setp_460 = _setp_460_bits != 0; + float nm_err_3 = ((_setp_460) ? nm_es_3 : nm_ex_3); + float _add_463; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_463) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_463; s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_432; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_432) : "f"(s_b_1), "f"(nm_t_4)); + float _sub_464; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_464) : "f"(s_b_1), "f"(nm_t_4)); float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_432), "f"(pb_1)); - float _sub_433; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_433) : "f"(pb_1), "f"(nm_t_4)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_464), "f"(pb_1)); + float _sub_465; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_465) : "f"(pb_1), "f"(nm_t_4)); float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_433), "f"(s_b_1)); - float _abs_435; - asm("abs.f32 %0, %1;" : "=f"(_abs_435) : "f"(s_b_1)); - float _abs_436; - asm("abs.f32 %0, %1;" : "=f"(_abs_436) : "f"(pb_1)); - uint32_t _setp_434_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_434_bits) : "f"(_abs_435), "f"(_abs_436)); - bool _setp_434 = _setp_434_bits != 0; - float nm_err_4 = ((_setp_434) ? nm_es_4 : nm_ex_4); - float _add_437; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_437) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_437; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_465), "f"(s_b_1)); + float _abs_467; + asm("abs.f32 %0, %1;" : "=f"(_abs_467) : "f"(s_b_1)); + float _abs_468; + asm("abs.f32 %0, %1;" : "=f"(_abs_468) : "f"(pb_1)); + uint32_t _setp_466_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_466_bits) : "f"(_abs_467), "f"(_abs_468)); + bool _setp_466 = _setp_466_bits != 0; + float nm_err_4 = ((_setp_466) ? nm_es_4 : nm_ex_4); + float _add_469; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_469) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_469; s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5104,8 +4883,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_438 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_438), "f"(off_acc[0]) : "memory"); + uint32_t _addr_470 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_470), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5129,320 +4908,124 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1514 = r_322[0]; - float gc_1515 = gn_1514 - 21.640425f; - int eb_1516 = 98304 + th * -32768; - int ew_1517 = eb_1516 >> 2; - int qbase_1518 = pool_f32_addr + (unsigned int)(eb_1516 + 12288 + dh_lane * 1024); - int sb8_1519 = 8 * th; - int gb8_1520 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1519 * 128); - int kb8_1521 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1519 * 128); - float dbp_1522[8]; - float _tmem_load_26[8]; - tmem_ld_x8(&_tmem_load_26[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float _tmem_load_27[8]; - tmem_ld_x8(&_tmem_load_27[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float _tmem_load_28[8]; - tmem_ld_x8(&_tmem_load_28[0], taddr + (unsigned int)row_base + (unsigned int)(8 * th)); - float btv_1523[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1523[0])), "=r"(*reinterpret_cast(&btv_1523[(0) + 1])), "=r"(*reinterpret_cast(&btv_1523[(0) + 2])), "=r"(*reinterpret_cast(&btv_1523[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1523[4])), "=r"(*reinterpret_cast(&btv_1523[(4) + 1])), "=r"(*reinterpret_cast(&btv_1523[(4) + 2])), "=r"(*reinterpret_cast(&btv_1523[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th) + 16))); + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_2 = r_280[0]; + float gc_2 = gn_2 - 21.640425f; + int eb_2 = 98304 + th * -32768; + int ew_2 = eb_2 >> 2; + int qbase_2 = pool_f32_addr + (unsigned int)(eb_2 + 12288 + dh_lane * 1024); + int sb8_2 = 8 * th; + int gb8_2 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_2 * 128); + int kb8_2 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_2 * 128); + int c_qa_1 = 8 * th; + int c_ka_1 = 8 * th; + int c_qd_1 = 128 + 8 * th; + int c_kd_1 = 144 + 8 * th; + int c_dk_1 = 8 * th; + int c_bp_1 = c_kd_1; + float s_run_2[1]; + s_run_2[0] = 0.0f; + #pragma unroll 1 + for (int k_2 = 0; k_2 < 8; k_2++) { + int j_2 = 7 - k_2; + float a_qd_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_1 + j_2))); + float a_kd_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_1 + j_2))); + float a_dk_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_1 + j_2))); + float btr_2[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_2[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_2 + j_2)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_2 = sb8_2 + j_2; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 * j_2 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_2 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 * j_2 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_2 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 * j_2 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_2 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_283[0] << 16); + float q_m_2 = ((t_2 >= clen) ? 0.0f : q_t_2); + float dqf_2 = pool_f32[ew_2 + 128 * j_2 + d_lane]; + float dkf_2 = pool_f32[ew_2 + (1024 + 128 * j_2) + d_lane]; + float dgf_2 = pool_f32[ew_2 + (2048 + 128 * j_2) + d_lane]; + float bt_t_2 = btr_2[0]; + float e_dg_2 = approx_exp2(g_t_2 - gc_2); + float dq2_2 = a_qd_2[0] * e_dg_2; + float dk2_2 = a_kd_2[0] * e_dg_2; + float dbv_2[1]; + dbv_2[0] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + a_dk_2[0]; + float qe_9 = q_m_2 * e_dg_2; + float kp_21 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, a_qd_2[0], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_2) * a_kd_2[0]); + float dg_o_2 = __fmaf_rn(q_m_2, dq2_2, __fmaf_rn(dk2b_2 - a_dk_2[0], k_t_2, cr_9)) + dgf_2; + s_run_2[0] = s_run_2[0] + dg_o_2; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_1 + j_2)), "f"(s_run_2[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_1 + j_2)), "f"(dbv_2[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(98304 - 32768 * th >> 2) + (128 * j_2 + d_lane)] = dq_o_2; + pool_f32[(98304 - 32768 * th >> 2) + (1024 + 128 * j_2 + d_lane)] = dk_o_2; + } + float tot_2 = s_run_2[0]; + float dbp_2[8]; + tmem_ld_x8(&dbp_2[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_1); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1524[1]; - s_run_1524[0] = 0.0f; - int t_16 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1517 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1517 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1517 + 2944 + d_lane]; - float bt_t_16 = btv_1523[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1514); - float e_dg_16 = approx_exp2(g_t_16 - gc_1515); - float dq2_16 = _tmem_load_26[7] * e_dg_16; - float dk2_16 = _tmem_load_27[7] * e_dg_16; - dbp_1522[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_28[7]; - float dg_o_16 = __fmaf_rn(q_t_1_24, dq2_16, (dk2b_16 - _tmem_load_28[7]) * k_t_16) + dgf_16; - s_run_1524[0] = s_run_1524[0] + dg_o_16; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_16; - pool_f32[(98304 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_16; - int t_17 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1517 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1517 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1517 + 2816 + d_lane]; - float bt_t_17 = btv_1523[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1514); - float e_dg_17 = approx_exp2(g_t_17 - gc_1515); - float dq2_17 = _tmem_load_26[6] * e_dg_17; - float dk2_17 = _tmem_load_27[6] * e_dg_17; - dbp_1522[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[6]; - float dg_o_17 = __fmaf_rn(q_t_1_25, dq2_17, (dk2b_17 - _tmem_load_28[6]) * k_t_17) + dgf_17; - s_run_1524[0] = s_run_1524[0] + dg_o_17; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_17; - pool_f32[(98304 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_17; - int t_18 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1517 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1517 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1517 + 2688 + d_lane]; - float bt_t_18 = btv_1523[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1514); - float e_dg_18 = approx_exp2(g_t_18 - gc_1515); - float dq2_18 = _tmem_load_26[5] * e_dg_18; - float dk2_18 = _tmem_load_27[5] * e_dg_18; - dbp_1522[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[5]; - float dg_o_18 = __fmaf_rn(q_t_1_26, dq2_18, (dk2b_18 - _tmem_load_28[5]) * k_t_18) + dgf_18; - s_run_1524[0] = s_run_1524[0] + dg_o_18; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_18; - pool_f32[(98304 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_18; - int t_19 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1517 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1517 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1517 + 2560 + d_lane]; - float bt_t_19 = btv_1523[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1514); - float e_dg_19 = approx_exp2(g_t_19 - gc_1515); - float dq2_19 = _tmem_load_26[4] * e_dg_19; - float dk2_19 = _tmem_load_27[4] * e_dg_19; - dbp_1522[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[4]; - float dg_o_19 = __fmaf_rn(q_t_1_27, dq2_19, (dk2b_19 - _tmem_load_28[4]) * k_t_19) + dgf_19; - s_run_1524[0] = s_run_1524[0] + dg_o_19; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_19; - pool_f32[(98304 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_19; - int t_20 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1517 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1517 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1517 + 2432 + d_lane]; - float bt_t_20 = btv_1523[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1514); - float e_dg_20 = approx_exp2(g_t_20 - gc_1515); - float dq2_20 = _tmem_load_26[3] * e_dg_20; - float dk2_20 = _tmem_load_27[3] * e_dg_20; - dbp_1522[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[3]; - float dg_o_20 = __fmaf_rn(q_t_1_28, dq2_20, (dk2b_20 - _tmem_load_28[3]) * k_t_20) + dgf_20; - s_run_1524[0] = s_run_1524[0] + dg_o_20; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_20; - pool_f32[(98304 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_20; - int t_21 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1517 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1517 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1517 + 2304 + d_lane]; - float bt_t_21 = btv_1523[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1514); - float e_dg_21 = approx_exp2(g_t_21 - gc_1515); - float dq2_21 = _tmem_load_26[2] * e_dg_21; - float dk2_21 = _tmem_load_27[2] * e_dg_21; - dbp_1522[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[2]; - float dg_o_21 = __fmaf_rn(q_t_1_29, dq2_21, (dk2b_21 - _tmem_load_28[2]) * k_t_21) + dgf_21; - s_run_1524[0] = s_run_1524[0] + dg_o_21; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_21; - pool_f32[(98304 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_21; - int t_22 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1517 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1517 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1517 + 2176 + d_lane]; - float bt_t_22 = btv_1523[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1514); - float e_dg_22 = approx_exp2(g_t_22 - gc_1515); - float dq2_22 = _tmem_load_26[1] * e_dg_22; - float dk2_22 = _tmem_load_27[1] * e_dg_22; - dbp_1522[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[1]; - float dg_o_22 = __fmaf_rn(q_t_1_30, dq2_22, (dk2b_22 - _tmem_load_28[1]) * k_t_22) + dgf_22; - s_run_1524[0] = s_run_1524[0] + dg_o_22; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_22; - pool_f32[(98304 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_22; - int t_23 = 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1517 + d_lane]; - float dkf_23 = pool_f32[ew_1517 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1517 + 2048 + d_lane]; - float bt_t_23 = btv_1523[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1514); - float e_dg_23 = approx_exp2(g_t_23 - gc_1515); - float dq2_23 = _tmem_load_26[0] * e_dg_23; - float dk2_23 = _tmem_load_27[0] * e_dg_23; - dbp_1522[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[0]; - float dg_o_23 = __fmaf_rn(q_t_1_31, dq2_23, (dk2b_23 - _tmem_load_28[0]) * k_t_23) + dgf_23; - s_run_1524[0] = s_run_1524[0] + dg_o_23; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + d_lane] = dq_o_23; - pool_f32[(98304 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_23; - float tot_1693 = s_run_1524[0]; - int b4_1694 = lane_1 >> 4 & 1; - int b3_1695 = lane_1 >> 3 & 1; - int b2_1696 = lane_1 >> 2 & 1; - float v4_1697[4]; - float send_12 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_439 = __shfl_xor_sync(0xFFFFFFFF, send_12, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_439; - float send_13 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_440 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_440; - float send_14 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_441 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_441; - float send_15 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_442 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_442; - float v2_1702[2]; - float send_16 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_443 = __shfl_xor_sync(0xFFFFFFFF, send_16, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_443; - float send_17 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_444 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_444; - float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_445 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_445; - float _shfl_xor_sync_446 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_446; - float _shfl_xor_sync_447 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_447; + int b4_2 = lane_1 >> 4 & 1; + int b3_2 = lane_1 >> 3 & 1; + int b2_2 = lane_1 >> 2 & 1; + float v4_2[4]; + float send_8 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); + float _shfl_xor_sync_471 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_471; + float send_9 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); + float _shfl_xor_sync_472 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_472; + float send_10 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); + float _shfl_xor_sync_473 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_473; + float send_11 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); + float _shfl_xor_sync_474 = __shfl_xor_sync(0xFFFFFFFF, send_11, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_474; + float v2_17[2]; + float send2_4 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); + float _shfl_xor_sync_475 = __shfl_xor_sync(0xFFFFFFFF, send2_4, 8); + v2_17[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_475; + float send2_5 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); + float _shfl_xor_sync_476 = __shfl_xor_sync(0xFFFFFFFF, send2_5, 8); + v2_17[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_476; + float send1_1 = ((b2_2 == 0) ? v2_17[1] : v2_17[0]); + float _shfl_xor_sync_477 = __shfl_xor_sync(0xFFFFFFFF, send1_1, 4); + float v1_2 = ((b2_2 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_477; + float _shfl_xor_sync_478 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_478; + float _shfl_xor_sync_479 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_479; if (8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + 8 * th) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(98304 - 32768 * th >> 2) + d_lane]; } @@ -5495,8 +5078,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_448 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_448), "f"(tot_1693) : "memory"); + uint32_t _addr_480 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_480), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5514,8 +5097,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other_1[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other_1[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi_2 = ((th == 1) ? tot_1693 : t_other_1[0]); - float t_lo_2 = ((th == 1) ? t_other_1[0] : tot_1693); + float t_hi_2 = ((th == 1) ? tot_2 : t_other_1[0]); + float t_lo_2 = ((th == 1) ? t_other_1[0] : tot_2); float offset_2 = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi_2); float sfx_2[8]; tmem_ld_x8(&sfx_2[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); @@ -5528,132 +5111,132 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_449 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_481 = make_float2(gacc[4], gacc[4]); float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_449), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_452; - asm("abs.f32 %0, %1;" : "=f"(_abs_452) : "f"(rp_z_11.x)); - float _neg_451; - asm("neg.f32 %0, %1;" : "=f"(_neg_451) : "f"(_abs_452)); - float _exp_450 = __nv_expf(_neg_451); - float _abs_455; - asm("abs.f32 %0, %1;" : "=f"(_abs_455) : "f"(rp_z_11.y)); - float _neg_454; - asm("neg.f32 %0, %1;" : "=f"(_neg_454) : "f"(_abs_455)); - float _exp_453 = __nv_expf(_neg_454); - float2 rp_e_11 = make_float2(_exp_450, _exp_453); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_481), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_484; + asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(rp_z_11.x)); + float _neg_483; + asm("neg.f32 %0, %1;" : "=f"(_neg_483) : "f"(_abs_484)); + float _exp_482 = __nv_expf(_neg_483); + float _abs_487; + asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(rp_z_11.y)); + float _neg_486; + asm("neg.f32 %0, %1;" : "=f"(_neg_486) : "f"(_abs_487)); + float _exp_485 = __nv_expf(_neg_486); + float2 rp_e_11 = make_float2(_exp_482, _exp_485); float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_456 = approx_rcp(rp_den_11.x); - float _rcp_457 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_458 = make_float2(_rcp_456, _rcp_457); + float _rcp_488 = approx_rcp(rp_den_11.x); + float _rcp_489 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_490 = make_float2(_rcp_488, _rcp_489); float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_490)); float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_459 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_491 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_459)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_491)); u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_460 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_492 = make_float2(gacc[4], gacc[4]); float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_460), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_463; - asm("abs.f32 %0, %1;" : "=f"(_abs_463) : "f"(rp_z_12.x)); - float _neg_462; - asm("neg.f32 %0, %1;" : "=f"(_neg_462) : "f"(_abs_463)); - float _exp_461 = __nv_expf(_neg_462); - float _abs_466; - asm("abs.f32 %0, %1;" : "=f"(_abs_466) : "f"(rp_z_12.y)); - float _neg_465; - asm("neg.f32 %0, %1;" : "=f"(_neg_465) : "f"(_abs_466)); - float _exp_464 = __nv_expf(_neg_465); - float2 rp_e_12 = make_float2(_exp_461, _exp_464); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_492), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_495; + asm("abs.f32 %0, %1;" : "=f"(_abs_495) : "f"(rp_z_12.x)); + float _neg_494; + asm("neg.f32 %0, %1;" : "=f"(_neg_494) : "f"(_abs_495)); + float _exp_493 = __nv_expf(_neg_494); + float _abs_498; + asm("abs.f32 %0, %1;" : "=f"(_abs_498) : "f"(rp_z_12.y)); + float _neg_497; + asm("neg.f32 %0, %1;" : "=f"(_neg_497) : "f"(_abs_498)); + float _exp_496 = __nv_expf(_neg_497); + float2 rp_e_12 = make_float2(_exp_493, _exp_496); float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_467 = approx_rcp(rp_den_12.x); - float _rcp_468 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_469 = make_float2(_rcp_467, _rcp_468); + float _rcp_499 = approx_rcp(rp_den_12.x); + float _rcp_500 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_501 = make_float2(_rcp_499, _rcp_500); float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_501)); float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_470 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_502 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_470)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_502)); u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_471 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_503 = make_float2(gacc[4], gacc[4]); float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_471), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_474; - asm("abs.f32 %0, %1;" : "=f"(_abs_474) : "f"(rp_z_13.x)); - float _neg_473; - asm("neg.f32 %0, %1;" : "=f"(_neg_473) : "f"(_abs_474)); - float _exp_472 = __nv_expf(_neg_473); - float _abs_477; - asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(rp_z_13.y)); - float _neg_476; - asm("neg.f32 %0, %1;" : "=f"(_neg_476) : "f"(_abs_477)); - float _exp_475 = __nv_expf(_neg_476); - float2 rp_e_13 = make_float2(_exp_472, _exp_475); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_503), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_506; + asm("abs.f32 %0, %1;" : "=f"(_abs_506) : "f"(rp_z_13.x)); + float _neg_505; + asm("neg.f32 %0, %1;" : "=f"(_neg_505) : "f"(_abs_506)); + float _exp_504 = __nv_expf(_neg_505); + float _abs_509; + asm("abs.f32 %0, %1;" : "=f"(_abs_509) : "f"(rp_z_13.y)); + float _neg_508; + asm("neg.f32 %0, %1;" : "=f"(_neg_508) : "f"(_abs_509)); + float _exp_507 = __nv_expf(_neg_508); + float2 rp_e_13 = make_float2(_exp_504, _exp_507); float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_478 = approx_rcp(rp_den_13.x); - float _rcp_479 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_480 = make_float2(_rcp_478, _rcp_479); + float _rcp_510 = approx_rcp(rp_den_13.x); + float _rcp_511 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_512 = make_float2(_rcp_510, _rcp_511); float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_480)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_512)); float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_481 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_513 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_481)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_513)); u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_482 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_514 = make_float2(gacc[4], gacc[4]); float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_482), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_485; - asm("abs.f32 %0, %1;" : "=f"(_abs_485) : "f"(rp_z_14.x)); - float _neg_484; - asm("neg.f32 %0, %1;" : "=f"(_neg_484) : "f"(_abs_485)); - float _exp_483 = __nv_expf(_neg_484); - float _abs_488; - asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(rp_z_14.y)); - float _neg_487; - asm("neg.f32 %0, %1;" : "=f"(_neg_487) : "f"(_abs_488)); - float _exp_486 = __nv_expf(_neg_487); - float2 rp_e_14 = make_float2(_exp_483, _exp_486); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_514), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_517; + asm("abs.f32 %0, %1;" : "=f"(_abs_517) : "f"(rp_z_14.x)); + float _neg_516; + asm("neg.f32 %0, %1;" : "=f"(_neg_516) : "f"(_abs_517)); + float _exp_515 = __nv_expf(_neg_516); + float _abs_520; + asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_14.y)); + float _neg_519; + asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); + float _exp_518 = __nv_expf(_neg_519); + float2 rp_e_14 = make_float2(_exp_515, _exp_518); float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_489 = approx_rcp(rp_den_14.x); - float _rcp_490 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_491 = make_float2(_rcp_489, _rcp_490); + float _rcp_521 = approx_rcp(rp_den_14.x); + float _rcp_522 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_523 = make_float2(_rcp_521, _rcp_522); float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_491)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_492 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_524 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_492)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_524)); u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -5668,54 +5251,54 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_493; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_493) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_493; + uint32_t _prmt_525; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_525) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_525; int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_494 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_494)); - *(reinterpret_cast(_gst_494) + (0)) = pq_12[0]; + auto _gst_526 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_526)); + *(reinterpret_cast(_gst_526) + (0)) = pq_12[0]; } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_495; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_495) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_495; + uint32_t _prmt_527; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_527) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_527; int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_496 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_496)); - *(reinterpret_cast(_gst_496) + (0)) = pq_13[0]; + auto _gst_528 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_528)); + *(reinterpret_cast(_gst_528) + (0)) = pq_13[0]; } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_497; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_497) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_497; + uint32_t _prmt_529; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_529) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_529; int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_498 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_498)); - *(reinterpret_cast(_gst_498) + (0)) = pq_14[0]; + auto _gst_530 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_530)); + *(reinterpret_cast(_gst_530) + (0)) = pq_14[0]; } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_499; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_499) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_499; + uint32_t _prmt_531; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_531) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_531; int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_500 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_500)); - *(reinterpret_cast(_gst_500) + (0)) = pq_15[0]; + auto _gst_532 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_532)); + *(reinterpret_cast(_gst_532) + (0)) = pq_15[0]; } } int t_a_24 = 8 * th; @@ -5780,47 +5363,47 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_501; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_501) : "f"(s_a_2), "f"(nm_t_5)); + float _sub_533; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_533) : "f"(s_a_2), "f"(nm_t_5)); float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_501), "f"(pa_2)); - float _sub_502; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_502) : "f"(pa_2), "f"(nm_t_5)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_533), "f"(pa_2)); + float _sub_534; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_534) : "f"(pa_2), "f"(nm_t_5)); float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_502), "f"(s_a_2)); - float _abs_504; - asm("abs.f32 %0, %1;" : "=f"(_abs_504) : "f"(s_a_2)); - float _abs_505; - asm("abs.f32 %0, %1;" : "=f"(_abs_505) : "f"(pa_2)); - uint32_t _setp_503_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_503_bits) : "f"(_abs_504), "f"(_abs_505)); - bool _setp_503 = _setp_503_bits != 0; - float nm_err_5 = ((_setp_503) ? nm_es_5 : nm_ex_5); - float _add_506; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_506) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_506; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_534), "f"(s_a_2)); + float _abs_536; + asm("abs.f32 %0, %1;" : "=f"(_abs_536) : "f"(s_a_2)); + float _abs_537; + asm("abs.f32 %0, %1;" : "=f"(_abs_537) : "f"(pa_2)); + uint32_t _setp_535_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_535_bits) : "f"(_abs_536), "f"(_abs_537)); + bool _setp_535 = _setp_535_bits != 0; + float nm_err_5 = ((_setp_535) ? nm_es_5 : nm_ex_5); + float _add_538; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_538) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_538; s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_507; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_507) : "f"(s_b_2), "f"(nm_t_6)); + float _sub_539; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_539) : "f"(s_b_2), "f"(nm_t_6)); float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_507), "f"(pb_2)); - float _sub_508; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_508) : "f"(pb_2), "f"(nm_t_6)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_539), "f"(pb_2)); + float _sub_540; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_540) : "f"(pb_2), "f"(nm_t_6)); float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_508), "f"(s_b_2)); - float _abs_510; - asm("abs.f32 %0, %1;" : "=f"(_abs_510) : "f"(s_b_2)); - float _abs_511; - asm("abs.f32 %0, %1;" : "=f"(_abs_511) : "f"(pb_2)); - uint32_t _setp_509_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_509_bits) : "f"(_abs_510), "f"(_abs_511)); - bool _setp_509 = _setp_509_bits != 0; - float nm_err_6 = ((_setp_509) ? nm_es_6 : nm_ex_6); - float _add_512; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_512) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_512; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_540), "f"(s_b_2)); + float _abs_542; + asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(s_b_2)); + float _abs_543; + asm("abs.f32 %0, %1;" : "=f"(_abs_543) : "f"(pb_2)); + uint32_t _setp_541_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_541_bits) : "f"(_abs_542), "f"(_abs_543)); + bool _setp_541 = _setp_541_bits != 0; + float nm_err_6 = ((_setp_541) ? nm_es_6 : nm_ex_6); + float _add_544; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_544) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_544; s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -5859,60 +5442,60 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_514; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_514) : "f"(ts_s), "f"(ts_bb)); - float _sub_513; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_513) : "f"(s_a_3), "f"(_sub_514)); - float _sub_515; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_515) : "f"(other[0]), "f"(ts_bb)); + float _sub_546; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_546) : "f"(ts_s), "f"(ts_bb)); + float _sub_545; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_545) : "f"(s_a_3), "f"(_sub_546)); + float _sub_547; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_547) : "f"(other[0]), "f"(ts_bb)); float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_513), "f"(_sub_515)); - float _add_516; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_516) : "f"(c_a_3), "f"(other[1])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_545), "f"(_sub_547)); + float _add_548; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_548) : "f"(c_a_3), "f"(other[1])); float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_516), "f"(ts_e)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_548), "f"(ts_e)); float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_518; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_518) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_517; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_517) : "f"(ts_s), "f"(_sub_518)); - float _sub_519; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_519) : "f"(pm_c), "f"(ts_bb_0)); + float _sub_550; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_550) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_549; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_549) : "f"(ts_s), "f"(_sub_550)); + float _sub_551; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_551) : "f"(pm_c), "f"(ts_bb_0)); float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_517), "f"(_sub_519)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_549), "f"(_sub_551)); s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_521; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_521) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_520; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_520) : "f"(s_b_3), "f"(_sub_521)); - float _sub_522; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_522) : "f"(other[2]), "f"(ts_bb_1)); + float _sub_553; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_553) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_552; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_552) : "f"(s_b_3), "f"(_sub_553)); + float _sub_554; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_554) : "f"(other[2]), "f"(ts_bb_1)); float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_520), "f"(_sub_522)); - float _add_523; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_523) : "f"(c_b_3), "f"(other[3])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_552), "f"(_sub_554)); + float _add_555; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_555) : "f"(c_b_3), "f"(other[3])); float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_523), "f"(ts_e_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_555), "f"(ts_e_1)); float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_525; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_525) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_524; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_524) : "f"(ts_s_1), "f"(_sub_525)); - float _sub_526; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_526) : "f"(pm_c_0), "f"(ts_bb_2)); + float _sub_557; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_557) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_556; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_556) : "f"(ts_s_1), "f"(_sub_557)); + float _sub_558; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_558) : "f"(pm_c_0), "f"(ts_bb_2)); float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_524), "f"(_sub_526)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_556), "f"(_sub_558)); s_b_3 = ts_s_2; c_b_3 = ts_e_2; gacc[0] = s_a_3; @@ -5930,10 +5513,10 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM } if ((lane_1 & 3) == 0) { int e_star = lane_1 >> 2 & 7; - pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_1706; - pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1484; + pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_2; + pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1; pool_f32[28160 + qd * 64 + 32 + 8 * th + e_star] = v1; - pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_1241; + pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_0; } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync 1, 256;" ::: "memory"); @@ -6023,41 +5606,41 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + for (int r_284 = 0; r_284 < 16; r_284++) { + int t_3 = r_284; + if (t_3 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6129,41 +5712,41 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_24 = 16 + r_348; - if (t_24 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + for (int r_285 = 0; r_285 < 16; r_285++) { + int t_4 = 16 + r_285; + if (t_4 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6279,41 +5862,41 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_25 = 32 + r_349; - if (t_25 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + for (int r_286 = 0; r_286 < 16; r_286++) { + int t_5 = 32 + r_286; + if (t_5 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6461,41 +6044,41 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_26 = 48 + r_350; - if (t_26 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + for (int r_287 = 0; r_287 < 16; r_287++) { + int t_6 = 48 + r_287; + if (t_6 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu index bc7a0fd7ea4..589b41afc9f 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2246,15 +2269,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2275,15 +2304,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2304,15 +2339,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2333,15 +2374,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2502,15 +2573,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2531,15 +2608,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2560,15 +2643,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2589,15 +2678,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2636,15 +2731,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2665,15 +2766,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2694,15 +2801,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2723,15 +2836,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2802,37 +2921,37 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2965,19 +3084,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; - float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0) + dgf_0; + float qe = q_t0_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr)) + dgf_0; dgo[0] = dg_o_0; int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_248 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_249 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -3003,19 +3127,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_1 = dk2_1 * bt_t_1; float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; - float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1) + dgf_1; + float qe_0 = q_t0_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0)) + dgf_1; dgo[1] = dg_o_1; int qoff_1 = t_1 * qrs; if (t_1 < clen) { { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_250 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_251 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3041,19 +3170,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; - float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2) + dgf_2; + float qe_1 = q_t0_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1)) + dgf_2; dgo[2] = dg_o_2; int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_252 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_253 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3079,19 +3213,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; - float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3) + dgf_3; + float qe_2 = q_t0_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2)) + dgf_3; dgo[3] = dg_o_3; int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_254 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_255 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3117,19 +3256,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; - float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4) + dgf_4; + float qe_3 = q_t0_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3)) + dgf_4; dgo[4] = dg_o_4; int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_256 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_257 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3155,19 +3299,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; - float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5) + dgf_5; + float qe_4 = q_t0_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4)) + dgf_5; dgo[5] = dg_o_5; int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_258 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_259 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3193,19 +3342,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; - float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6) + dgf_6; + float qe_5 = q_t0_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5)) + dgf_6; dgo[6] = dg_o_6; int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_260 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_261 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3231,19 +3385,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; - float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7) + dgf_7; + float qe_6 = q_t0_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6)) + dgf_7; dgo[7] = dg_o_7; int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_262 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_263 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_7; } } float s_run[1]; @@ -3272,31 +3431,31 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3355,19 +3514,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; - float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8) + dgf_8; + float qe_7 = q_t0_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7)) + dgf_8; dgo_0[0] = dg_o_8; int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_273 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_274 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3393,19 +3557,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; - float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9) + dgf_9; + float qe_8 = q_t0_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8)) + dgf_9; dgo_0[1] = dg_o_9; int qoff_9 = t_9 * qrs; if (t_9 < clen) { { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_275 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_276 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3431,19 +3600,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; - float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10) + dgf_10; + float qe_9 = q_t0_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9)) + dgf_10; dgo_0[2] = dg_o_10; int qoff_10 = t_10 * qrs; if (t_10 < clen) { { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_277 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_278 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3469,19 +3643,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; - float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11) + dgf_11; + float qe_10 = q_t0_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10)) + dgf_11; dgo_0[3] = dg_o_11; int qoff_11 = t_11 * qrs; if (t_11 < clen) { { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_279 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_280 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3507,19 +3686,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; - float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12) + dgf_12; + float qe_11 = q_t0_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11)) + dgf_12; dgo_0[4] = dg_o_12; int qoff_12 = t_12 * qrs; if (t_12 < clen) { { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_281 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_282 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3545,19 +3729,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; - float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13) + dgf_13; + float qe_12 = q_t0_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12)) + dgf_13; dgo_0[5] = dg_o_13; int qoff_13 = t_13 * qrs; if (t_13 < clen) { { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_283 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_284 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3583,19 +3772,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; - float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14) + dgf_14; + float qe_13 = q_t0_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13)) + dgf_14; dgo_0[6] = dg_o_14; int qoff_14 = t_14 * qrs; if (t_14 < clen) { { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_285 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_286 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3621,19 +3815,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; - float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15) + dgf_15; + float qe_14 = q_t0_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14)) + dgf_15; dgo_0[7] = dg_o_15; int qoff_15 = t_15 * qrs; if (t_15 < clen) { { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_287 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_288 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_15; } } float s_run_0[1]; @@ -3662,36 +3861,36 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_289; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_290; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_291; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_292; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_293; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_294; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -3709,9 +3908,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_0 = offset + sfx[0]; if (t_16 < clen) { { - auto _gst_267 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_0; + auto _gst_299 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_0; } } int t_17 = 48 + 8 * th + 1; @@ -3719,9 +3918,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_1 = offset + sfx[1]; if (t_17 < clen) { { - auto _gst_268 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1; + auto _gst_300 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1; } } int t_18 = 48 + 8 * th + 2; @@ -3729,9 +3928,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_2 = offset + sfx[2]; if (t_18 < clen) { { - auto _gst_269 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_2; + auto _gst_301 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_2; } } int t_19 = 48 + 8 * th + 3; @@ -3739,9 +3938,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_3 = offset + sfx[3]; if (t_19 < clen) { { - auto _gst_270 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_3; + auto _gst_302 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_3; } } int t_20 = 48 + 8 * th + 4; @@ -3749,9 +3948,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_4 = offset + sfx[4]; if (t_20 < clen) { { - auto _gst_271 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_4; + auto _gst_303 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_4; } } int t_21 = 48 + 8 * th + 5; @@ -3759,9 +3958,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_5 = offset + sfx[5]; if (t_21 < clen) { { - auto _gst_272 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_5; + auto _gst_304 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_5; } } int t_22 = 48 + 8 * th + 6; @@ -3769,9 +3968,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_6 = offset + sfx[6]; if (t_22 < clen) { { - auto _gst_273 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_6; + auto _gst_305 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_6; } } int t_23 = 48 + 8 * th + 7; @@ -3779,9 +3978,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_7 = offset + sfx[7]; if (t_23 < clen) { { - auto _gst_274 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_7; + auto _gst_306 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3799,9 +3998,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_8 = offset_0 + sfx_0[0]; if (t_24 < clen) { { - auto _gst_275 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_8; + auto _gst_307 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_8; } } int t_25 = 32 + 8 * th + 1; @@ -3809,9 +4008,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_9 = offset_0 + sfx_0[1]; if (t_25 < clen) { { - auto _gst_276 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_9; + auto _gst_308 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_9; } } int t_26 = 32 + 8 * th + 2; @@ -3819,9 +4018,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_10 = offset_0 + sfx_0[2]; if (t_26 < clen) { { - auto _gst_277 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_10; + auto _gst_309 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_10; } } int t_27 = 32 + 8 * th + 3; @@ -3829,9 +4028,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_11 = offset_0 + sfx_0[3]; if (t_27 < clen) { { - auto _gst_278 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_11; + auto _gst_310 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_11; } } int t_28 = 32 + 8 * th + 4; @@ -3839,9 +4038,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_12 = offset_0 + sfx_0[4]; if (t_28 < clen) { { - auto _gst_279 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_12; + auto _gst_311 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_12; } } int t_29 = 32 + 8 * th + 5; @@ -3849,9 +4048,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_13 = offset_0 + sfx_0[5]; if (t_29 < clen) { { - auto _gst_280 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_13; + auto _gst_312 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_13; } } int t_30 = 32 + 8 * th + 6; @@ -3859,9 +4058,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_14 = offset_0 + sfx_0[6]; if (t_30 < clen) { { - auto _gst_281 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_14; + auto _gst_313 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_14; } } int t_31 = 32 + 8 * th + 7; @@ -3869,15 +4068,15 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_15 = offset_0 + sfx_0[7]; if (t_31 < clen) { { - auto _gst_282 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_15; + auto _gst_314 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3936,19 +4135,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; - float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16) + dgf_16; + float qe_15 = q_t0_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15)) + dgf_16; dgo_1[0] = dg_o_16; int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_284 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_316 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_285 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_317 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_33 = 16 + 8 * th + 1; @@ -3974,19 +4178,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; - float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17) + dgf_17; + float qe_16 = q_t0_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16)) + dgf_17; dgo_1[1] = dg_o_17; int qoff_17 = t_33 * qrs; if (t_33 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_34 = 16 + 8 * th + 2; @@ -4012,19 +4221,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; - float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18) + dgf_18; + float qe_17 = q_t0_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17)) + dgf_18; dgo_1[2] = dg_o_18; int qoff_18 = t_34 * qrs; if (t_34 < clen) { { - auto _gst_288 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_320 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_289 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_321 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_35 = 16 + 8 * th + 3; @@ -4050,19 +4264,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; - float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19) + dgf_19; + float qe_18 = q_t0_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18)) + dgf_19; dgo_1[3] = dg_o_19; int qoff_19 = t_35 * qrs; if (t_35 < clen) { { - auto _gst_290 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_322 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_291 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_323 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_36 = 16 + 8 * th + 4; @@ -4088,19 +4307,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; - float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20) + dgf_20; + float qe_19 = q_t0_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19)) + dgf_20; dgo_1[4] = dg_o_20; int qoff_20 = t_36 * qrs; if (t_36 < clen) { { - auto _gst_292 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_324 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_293 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_325 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_37 = 16 + 8 * th + 5; @@ -4126,19 +4350,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; - float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21) + dgf_21; + float qe_20 = q_t0_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20)) + dgf_21; dgo_1[5] = dg_o_21; int qoff_21 = t_37 * qrs; if (t_37 < clen) { { - auto _gst_294 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_326 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_295 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_327 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_38 = 16 + 8 * th + 6; @@ -4164,19 +4393,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; - float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22) + dgf_22; + float qe_21 = q_t0_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21)) + dgf_22; dgo_1[6] = dg_o_22; int qoff_22 = t_38 * qrs; if (t_38 < clen) { { - auto _gst_296 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_328 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_297 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_329 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_39 = 16 + 8 * th + 7; @@ -4202,19 +4436,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; - float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23) + dgf_23; + float qe_22 = q_t0_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22)) + dgf_23; dgo_1[7] = dg_o_23; int qoff_23 = t_39 * qrs; if (t_39 < clen) { { - auto _gst_298 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_330 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_299 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_331 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_23; } } float s_run_1[1]; @@ -4243,37 +4482,37 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_332; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_333; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_334; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_335; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_336; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_337; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4291,9 +4530,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_16 = offset_1 + sfx_1[0]; if (t_40 < clen) { { - auto _gst_310 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_16; + auto _gst_342 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_16; } } int t_41 = 16 + 8 * th + 1; @@ -4301,9 +4540,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_17 = offset_1 + sfx_1[1]; if (t_41 < clen) { { - auto _gst_311 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_17; + auto _gst_343 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_17; } } int t_42 = 16 + 8 * th + 2; @@ -4311,9 +4550,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_18 = offset_1 + sfx_1[2]; if (t_42 < clen) { { - auto _gst_312 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_18; + auto _gst_344 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_18; } } int t_43 = 16 + 8 * th + 3; @@ -4321,9 +4560,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_19 = offset_1 + sfx_1[3]; if (t_43 < clen) { { - auto _gst_313 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_19; + auto _gst_345 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_19; } } int t_44 = 16 + 8 * th + 4; @@ -4331,9 +4570,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_20 = offset_1 + sfx_1[4]; if (t_44 < clen) { { - auto _gst_314 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_20; + auto _gst_346 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_20; } } int t_45 = 16 + 8 * th + 5; @@ -4341,9 +4580,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_21 = offset_1 + sfx_1[5]; if (t_45 < clen) { { - auto _gst_315 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_21; + auto _gst_347 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_21; } } int t_46 = 16 + 8 * th + 6; @@ -4351,9 +4590,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_22 = offset_1 + sfx_1[6]; if (t_46 < clen) { { - auto _gst_316 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_22; + auto _gst_348 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_22; } } int t_47 = 16 + 8 * th + 7; @@ -4361,15 +4600,15 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_23 = offset_1 + sfx_1[7]; if (t_47 < clen) { { - auto _gst_317 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_23; + auto _gst_349 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4422,19 +4661,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; - float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24) + dgf_24; + float qe_23 = q_t0_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23)) + dgf_24; dgo_2[0] = dg_o_24; int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_319 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_351 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_320 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_352 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_49 = 8 * th + 1; @@ -4460,19 +4704,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_25 = dk2_25 * bt_t_25; float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; - float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25) + dgf_25; + float qe_24 = q_t0_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24)) + dgf_25; dgo_2[1] = dg_o_25; int qoff_25 = t_49 * qrs; if (t_49 < clen) { { - auto _gst_321 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_353 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_322 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_354 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_50 = 8 * th + 2; @@ -4498,19 +4747,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_26 = dk2_26 * bt_t_26; float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; - float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26) + dgf_26; + float qe_25 = q_t0_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25)) + dgf_26; dgo_2[2] = dg_o_26; int qoff_26 = t_50 * qrs; if (t_50 < clen) { { - auto _gst_323 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_355 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_324 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_356 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_51 = 8 * th + 3; @@ -4536,19 +4790,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_27 = dk2_27 * bt_t_27; float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; - float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27) + dgf_27; + float qe_26 = q_t0_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26)) + dgf_27; dgo_2[3] = dg_o_27; int qoff_27 = t_51 * qrs; if (t_51 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_52 = 8 * th + 4; @@ -4574,19 +4833,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_28 = dk2_28 * bt_t_28; float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; - float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28) + dgf_28; + float qe_27 = q_t0_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27)) + dgf_28; dgo_2[4] = dg_o_28; int qoff_28 = t_52 * qrs; if (t_52 < clen) { { - auto _gst_327 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_359 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_328 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_360 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_53 = 8 * th + 5; @@ -4612,19 +4876,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_29 = dk2_29 * bt_t_29; float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; - float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29) + dgf_29; + float qe_28 = q_t0_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28)) + dgf_29; dgo_2[5] = dg_o_29; int qoff_29 = t_53 * qrs; if (t_53 < clen) { { - auto _gst_329 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_361 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_330 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_362 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_54 = 8 * th + 6; @@ -4650,19 +4919,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_30 = dk2_30 * bt_t_30; float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; - float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30) + dgf_30; + float qe_29 = q_t0_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29)) + dgf_30; dgo_2[6] = dg_o_30; int qoff_30 = t_54 * qrs; if (t_54 < clen) { { - auto _gst_331 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_363 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_332 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_364 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_55 = 8 * th + 7; @@ -4688,19 +4962,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_31 = dk2_31 * bt_t_31; float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; - float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31) + dgf_31; + float qe_30 = q_t0_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30)) + dgf_31; dgo_2[7] = dg_o_31; int qoff_31 = t_55 * qrs; if (t_55 < clen) { { - auto _gst_333 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_365 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_334 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_366 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_31; } } float s_run_2[1]; @@ -4729,35 +5008,35 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_367; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_368; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_369; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_370; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_371; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_372; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_2) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4775,9 +5054,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_24 = offset_2 + sfx_2[0]; if (t_56 < clen) { { - auto _gst_345 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_24; + auto _gst_377 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_24; } } int t_57 = 8 * th + 1; @@ -4785,9 +5064,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_25 = offset_2 + sfx_2[1]; if (t_57 < clen) { { - auto _gst_346 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_25; + auto _gst_378 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_25; } } int t_58 = 8 * th + 2; @@ -4795,9 +5074,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_26 = offset_2 + sfx_2[2]; if (t_58 < clen) { { - auto _gst_347 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_26; + auto _gst_379 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_26; } } int t_59 = 8 * th + 3; @@ -4805,9 +5084,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_27 = offset_2 + sfx_2[3]; if (t_59 < clen) { { - auto _gst_348 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_27; + auto _gst_380 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_27; } } int t_60 = 8 * th + 4; @@ -4815,9 +5094,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_28 = offset_2 + sfx_2[4]; if (t_60 < clen) { { - auto _gst_349 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_28; + auto _gst_381 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_28; } } int t_61 = 8 * th + 5; @@ -4825,9 +5104,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_29 = offset_2 + sfx_2[5]; if (t_61 < clen) { { - auto _gst_350 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_29; + auto _gst_382 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_29; } } int t_62 = 8 * th + 6; @@ -4835,9 +5114,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_30 = offset_2 + sfx_2[6]; if (t_62 < clen) { { - auto _gst_351 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_30; + auto _gst_383 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_30; } } int t_63 = 8 * th + 7; @@ -4845,9 +5124,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_31 = offset_2 + sfx_2[7]; if (t_63 < clen) { { - auto _gst_352 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_31; + auto _gst_384 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_31; } } off_acc[0] = off_acc[0] + t_hi_2 + t_lo_2; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu index 8a6efdd1ecf..e9d1eba4441 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -199,6 +200,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2267,12 +2284,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_0); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_0 : __fmaf_rn(dl, _rcp_170, e2_0)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2296,12 +2319,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_1); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_1, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_1, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_1 : __fmaf_rn(dl_1, _rcp_174, e2_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2325,12 +2354,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_2); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_2, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_2, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_2 : __fmaf_rn(dl_3, _rcp_178, e2_2)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2354,12 +2389,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_3); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_3, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_3, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_3 : __fmaf_rn(dl_5, _rcp_182, e2_3)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2395,12 +2436,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2420,12 +2467,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2445,12 +2498,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2470,12 +2529,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2523,12 +2588,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2552,12 +2623,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_1 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2581,12 +2658,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2610,12 +2693,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2657,12 +2746,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2686,12 +2781,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2715,12 +2816,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2744,12 +2851,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2820,37 +2933,37 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 d2_56 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_56.x); float e1_12 = approx_exp2(d2_56.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_57.x); float e1_13 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_58.x); float e1_14 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_59.x); float e1_15 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2862,29 +2975,29 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2892,29 +3005,29 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2933,409 +3046,145 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float r_247[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_247[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 4096 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 4096 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); float gn = r_247[0]; - float gc_888 = gn - 21.640425f; + float gc_0 = gn - 21.640425f; int eb = 98304 + th * -32768; int ew = eb >> 2; - int qbase_889 = pool_f32_addr + (unsigned int)(eb + 12288 + dh_lane * 1024); + int qbase_0 = pool_f32_addr + (unsigned int)(eb + 12288 + dh_lane * 1024); int sb8 = 32 + 8 * th; int gb8 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8 * 128); int kb8 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8 * 128); - float dbp[8]; - float _tmem_load_7[8]; - tmem_ld_x8(&_tmem_load_7[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float _tmem_load_8[8]; - tmem_ld_x8(&_tmem_load_8[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float _tmem_load_9[8]; - tmem_ld_x8(&_tmem_load_9[0], taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)); - float _tmem_load_10[8]; - tmem_ld_x8(&_tmem_load_10[0], taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th)); - float _tmem_load_11[8]; - tmem_ld_x8(&_tmem_load_11[0], taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th)); - float btv[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[0])), "=r"(*reinterpret_cast(&btv[(0) + 1])), "=r"(*reinterpret_cast(&btv[(0) + 2])), "=r"(*reinterpret_cast(&btv[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[4])), "=r"(*reinterpret_cast(&btv[(4) + 1])), "=r"(*reinterpret_cast(&btv[(4) + 2])), "=r"(*reinterpret_cast(&btv[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int c_qa = 128 + 8 * th; + int c_ka = 144 + 8 * th; + int c_qd = 192 + 8 * th; + int c_kd = 208 + 8 * th; + int c_dk = 32 + 8 * th; + int c_bp = c_ka; float s_run[1]; s_run[0] = 0.0f; - int t_1_0 = 32 + 8 * th + 7; - float r_248[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_0 = r_248[0]; - unsigned int r_249[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_0 = __uint_as_float(r_249[0] << 16); - unsigned int r_250[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_0 = __uint_as_float(r_250[0] << 16); - float q_t_2_0 = ((t_1_0 >= clen) ? 0.0f : q_t_1_0); - float dqf_1_0 = pool_f32[ew + 896 + d_lane]; - float dkf_1_0 = pool_f32[ew + 1920 + d_lane]; - float dgf_1_0 = pool_f32[ew + 2944 + d_lane]; - float bt_t_1_0 = btv[7]; - float e_off_1_0 = approx_exp2(g_t_1_0 - gn); - float e_dg_1_0 = approx_exp2(g_t_1_0 - gc_888); - float dq2_1_0 = __fmaf_rn(_tmem_load_7[7], e_off_1_0, _tmem_load_9[7] * e_dg_1_0); - float dk2_1_0 = __fmaf_rn(_tmem_load_8[7], e_off_1_0, _tmem_load_10[7] * e_dg_1_0); - dbp[7] = dk2_1_0 * k_t_1_0; - float dk2b_1_0 = dk2_1_0 * bt_t_1_0; - float dq_o_1_0 = dq2_1_0 + dqf_1_0; - float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; - s_run[0] = s_run[0] + dg_o_1_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 7)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_0 = t_1_0 * qrs; - if (t_1_0 < clen) { - { - auto _gst_216 = dq_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_1_0; - } - { - auto _gst_217 = dk_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_1_0; - } - } - int t_1_1 = 32 + 8 * th + 6; - float r_251[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_1 = r_251[0]; - unsigned int r_252[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((kb8 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_1 = __uint_as_float(r_252[0] << 16); - unsigned int r_253[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_1 = __uint_as_float(r_253[0] << 16); - float q_t_2_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_1); - float dqf_1_1 = pool_f32[ew + 768 + d_lane]; - float dkf_1_1 = pool_f32[ew + 1792 + d_lane]; - float dgf_1_1 = pool_f32[ew + 2816 + d_lane]; - float bt_t_1_1 = btv[6]; - float e_off_1_1 = approx_exp2(g_t_1_1 - gn); - float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_888); - float dq2_1_1 = __fmaf_rn(_tmem_load_7[6], e_off_1_1, _tmem_load_9[6] * e_dg_1_1); - float dk2_1_1 = __fmaf_rn(_tmem_load_8[6], e_off_1_1, _tmem_load_10[6] * e_dg_1_1); - dbp[6] = dk2_1_1 * k_t_1_1; - float dk2b_1_1 = dk2_1_1 * bt_t_1_1; - float dq_o_1_1 = dq2_1_1 + dqf_1_1; - float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; - s_run[0] = s_run[0] + dg_o_1_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 6)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_1 = t_1_1 * qrs; - if (t_1_1 < clen) { - { - auto _gst_218 = dq_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1_1; - } - { - auto _gst_219 = dk_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1_1; - } - } - int t_1_2 = 32 + 8 * th + 5; - float r_254[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_2 = r_254[0]; - unsigned int r_255[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((kb8 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_2 = __uint_as_float(r_255[0] << 16); - unsigned int r_256[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_2 = __uint_as_float(r_256[0] << 16); - float q_t_2_2 = ((t_1_2 >= clen) ? 0.0f : q_t_1_2); - float dqf_1_2 = pool_f32[ew + 640 + d_lane]; - float dkf_1_2 = pool_f32[ew + 1664 + d_lane]; - float dgf_1_2 = pool_f32[ew + 2688 + d_lane]; - float bt_t_1_2 = btv[5]; - float e_off_1_2 = approx_exp2(g_t_1_2 - gn); - float e_dg_1_2 = approx_exp2(g_t_1_2 - gc_888); - float dq2_1_2 = __fmaf_rn(_tmem_load_7[5], e_off_1_2, _tmem_load_9[5] * e_dg_1_2); - float dk2_1_2 = __fmaf_rn(_tmem_load_8[5], e_off_1_2, _tmem_load_10[5] * e_dg_1_2); - dbp[5] = dk2_1_2 * k_t_1_2; - float dk2b_1_2 = dk2_1_2 * bt_t_1_2; - float dq_o_1_2 = dq2_1_2 + dqf_1_2; - float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; - s_run[0] = s_run[0] + dg_o_1_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 5)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_2 = t_1_2 * qrs; - if (t_1_2 < clen) { - { - auto _gst_220 = dq_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1_2; - } - { - auto _gst_221 = dk_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1_2; - } - } - int t_1_3 = 32 + 8 * th + 4; - float r_257[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_3 = r_257[0]; - unsigned int r_258[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((kb8 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_3 = __uint_as_float(r_258[0] << 16); - unsigned int r_259[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_3 = __uint_as_float(r_259[0] << 16); - float q_t_2_3 = ((t_1_3 >= clen) ? 0.0f : q_t_1_3); - float dqf_1_3 = pool_f32[ew + 512 + d_lane]; - float dkf_1_3 = pool_f32[ew + 1536 + d_lane]; - float dgf_1_3 = pool_f32[ew + 2560 + d_lane]; - float bt_t_1_3 = btv[4]; - float e_off_1_3 = approx_exp2(g_t_1_3 - gn); - float e_dg_1_3 = approx_exp2(g_t_1_3 - gc_888); - float dq2_1_3 = __fmaf_rn(_tmem_load_7[4], e_off_1_3, _tmem_load_9[4] * e_dg_1_3); - float dk2_1_3 = __fmaf_rn(_tmem_load_8[4], e_off_1_3, _tmem_load_10[4] * e_dg_1_3); - dbp[4] = dk2_1_3 * k_t_1_3; - float dk2b_1_3 = dk2_1_3 * bt_t_1_3; - float dq_o_1_3 = dq2_1_3 + dqf_1_3; - float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; - s_run[0] = s_run[0] + dg_o_1_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 4)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_3 = t_1_3 * qrs; - if (t_1_3 < clen) { - { - auto _gst_222 = dq_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_1_3; - } - { - auto _gst_223 = dk_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_1_3; - } - } - int t_1_4 = 32 + 8 * th + 3; - float r_260[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_4 = r_260[0]; - unsigned int r_261[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((kb8 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_4 = __uint_as_float(r_261[0] << 16); - unsigned int r_262[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_4 = __uint_as_float(r_262[0] << 16); - float q_t_2_4 = ((t_1_4 >= clen) ? 0.0f : q_t_1_4); - float dqf_1_4 = pool_f32[ew + 384 + d_lane]; - float dkf_1_4 = pool_f32[ew + 1408 + d_lane]; - float dgf_1_4 = pool_f32[ew + 2432 + d_lane]; - float bt_t_1_4 = btv[3]; - float e_off_1_4 = approx_exp2(g_t_1_4 - gn); - float e_dg_1_4 = approx_exp2(g_t_1_4 - gc_888); - float dq2_1_4 = __fmaf_rn(_tmem_load_7[3], e_off_1_4, _tmem_load_9[3] * e_dg_1_4); - float dk2_1_4 = __fmaf_rn(_tmem_load_8[3], e_off_1_4, _tmem_load_10[3] * e_dg_1_4); - dbp[3] = dk2_1_4 * k_t_1_4; - float dk2b_1_4 = dk2_1_4 * bt_t_1_4; - float dq_o_1_4 = dq2_1_4 + dqf_1_4; - float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; - s_run[0] = s_run[0] + dg_o_1_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 3)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_4 = t_1_4 * qrs; - if (t_1_4 < clen) { - { - auto _gst_224 = dq_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_1_4; - } - { - auto _gst_225 = dk_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_1_4; - } - } - int t_1_5 = 32 + 8 * th + 2; - float r_263[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_5 = r_263[0]; - unsigned int r_264[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((kb8 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_5 = __uint_as_float(r_264[0] << 16); - unsigned int r_265[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_5 = __uint_as_float(r_265[0] << 16); - float q_t_2_5 = ((t_1_5 >= clen) ? 0.0f : q_t_1_5); - float dqf_1_5 = pool_f32[ew + 256 + d_lane]; - float dkf_1_5 = pool_f32[ew + 1280 + d_lane]; - float dgf_1_5 = pool_f32[ew + 2304 + d_lane]; - float bt_t_1_5 = btv[2]; - float e_off_1_5 = approx_exp2(g_t_1_5 - gn); - float e_dg_1_5 = approx_exp2(g_t_1_5 - gc_888); - float dq2_1_5 = __fmaf_rn(_tmem_load_7[2], e_off_1_5, _tmem_load_9[2] * e_dg_1_5); - float dk2_1_5 = __fmaf_rn(_tmem_load_8[2], e_off_1_5, _tmem_load_10[2] * e_dg_1_5); - dbp[2] = dk2_1_5 * k_t_1_5; - float dk2b_1_5 = dk2_1_5 * bt_t_1_5; - float dq_o_1_5 = dq2_1_5 + dqf_1_5; - float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; - s_run[0] = s_run[0] + dg_o_1_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 2)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_5 = t_1_5 * qrs; - if (t_1_5 < clen) { - { - auto _gst_226 = dq_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_1_5; - } - { - auto _gst_227 = dk_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_1_5; - } - } - int t_1_6 = 32 + 8 * th + 1; - float r_266[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_6 = r_266[0]; - unsigned int r_267[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_6 = __uint_as_float(r_267[0] << 16); - unsigned int r_268[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_6 = __uint_as_float(r_268[0] << 16); - float q_t_2_6 = ((t_1_6 >= clen) ? 0.0f : q_t_1_6); - float dqf_1_6 = pool_f32[ew + 128 + d_lane]; - float dkf_1_6 = pool_f32[ew + 1152 + d_lane]; - float dgf_1_6 = pool_f32[ew + 2176 + d_lane]; - float bt_t_1_6 = btv[1]; - float e_off_1_6 = approx_exp2(g_t_1_6 - gn); - float e_dg_1_6 = approx_exp2(g_t_1_6 - gc_888); - float dq2_1_6 = __fmaf_rn(_tmem_load_7[1], e_off_1_6, _tmem_load_9[1] * e_dg_1_6); - float dk2_1_6 = __fmaf_rn(_tmem_load_8[1], e_off_1_6, _tmem_load_10[1] * e_dg_1_6); - dbp[1] = dk2_1_6 * k_t_1_6; - float dk2b_1_6 = dk2_1_6 * bt_t_1_6; - float dq_o_1_6 = dq2_1_6 + dqf_1_6; - float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; - s_run[0] = s_run[0] + dg_o_1_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 1)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_6 = t_1_6 * qrs; - if (t_1_6 < clen) { - { - auto _gst_228 = dq_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_1_6; - } - { - auto _gst_229 = dk_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_1_6; - } - } - int t_1_7 = 32 + 8 * th; - float r_269[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_7 = r_269[0]; - unsigned int r_270[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((kb8 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_7 = __uint_as_float(r_270[0] << 16); - unsigned int r_271[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_7 = __uint_as_float(r_271[0] << 16); - float q_t_2_7 = ((t_1_7 >= clen) ? 0.0f : q_t_1_7); - float dqf_1_7 = pool_f32[ew + d_lane]; - float dkf_1_7 = pool_f32[ew + 1024 + d_lane]; - float dgf_1_7 = pool_f32[ew + 2048 + d_lane]; - float bt_t_1_7 = btv[0]; - float e_off_1_7 = approx_exp2(g_t_1_7 - gn); - float e_dg_1_7 = approx_exp2(g_t_1_7 - gc_888); - float dq2_1_7 = __fmaf_rn(_tmem_load_7[0], e_off_1_7, _tmem_load_9[0] * e_dg_1_7); - float dk2_1_7 = __fmaf_rn(_tmem_load_8[0], e_off_1_7, _tmem_load_10[0] * e_dg_1_7); - dbp[0] = dk2_1_7 * k_t_1_7; - float dk2b_1_7 = dk2_1_7 * bt_t_1_7; - float dq_o_1_7 = dq2_1_7 + dqf_1_7; - float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; - s_run[0] = s_run[0] + dg_o_1_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_7 = t_1_7 * qrs; - if (t_1_7 < clen) { - { - auto _gst_230 = dq_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_1_7; - } - { - auto _gst_231 = dk_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_1_7; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j = 7 - k; + float a_qd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j))); + float a_kd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd + j))); + float a_dk[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk + j))); + float a_qa[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa + j))); + float a_ka[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka + j))); + float btr[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8 + j)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8 + j; + float r_248[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_248[0]; + unsigned int r_249[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_249[0] << 16); + unsigned int r_250[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_250[0] << 16); + float q_m = ((t >= clen) ? 0.0f : q_t); + float dqf = pool_f32[ew + 128 * j + d_lane]; + float dkf = pool_f32[ew + (1024 + 128 * j) + d_lane]; + float dgf = pool_f32[ew + (2048 + 128 * j) + d_lane]; + float bt_t = btr[0]; + float e_dg = approx_exp2(g_t - gc_0); + float e_off = approx_exp2(g_t - gn); + float dq2 = __fmaf_rn(a_qa[0], e_off, a_qd[0] * e_dg); + float dk2 = __fmaf_rn(a_ka[0], e_off, a_kd[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + a_dk[0]; + float qe = q_m * e_dg; + float kp = k_t * bt_t; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, a_qd[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg) * a_kd[0]); + float dg_o = __fmaf_rn(q_m, dq2, __fmaf_rn(dk2b - a_dk[0], k_t, cr)) + dgf; + s_run[0] = s_run[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j)), "f"(s_run[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff = t * qrs; + if (t < clen) { + { + auto _gst_248 = dq_base + qoff; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o; + } + { + auto _gst_249 = dk_base + qoff; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o; + } } } float tot = s_run[0]; + float dbp[8]; + tmem_ld_x8(&dbp[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); int b4 = lane_1 >> 4 & 1; int b3 = lane_1 >> 3 & 1; int b2 = lane_1 >> 2 & 1; float v4[4]; - float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; - float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; - float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; - float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; - float v2_1044[2]; - float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; - float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; - float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_250; + float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_251; + float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_252; + float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_253; + float v2[2]; + float send2_0 = ((b3 == 0) ? v4[2] : v4[0]); + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send2_0, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_254; + float send2_1 = ((b3 == 0) ? v4[3] : v4[1]); + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send2_1, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_255; + float send1 = ((b2 == 0) ? v2[1] : v2[0]); + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send1, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_256; + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_257; + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_258; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3346,412 +3195,148 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM if (lane_1 == 17) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(dt_bias + (hv * 128 + 32 * qd)))); } mbarrier_wait(epi_full_addr + (th) * 8, 1); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_272[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1049 = r_272[0]; - float gc_1050 = gn_1049 - 21.640425f; - int eb_1051 = 98304 + th * -32768; - int ew_1052 = eb_1051 >> 2; - int qbase_1053 = pool_f32_addr + (unsigned int)(eb_1051 + 12288 + dh_lane * 1024); - int sb8_1054 = 48 + 8 * th; - int gb8_1055 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1054 * 128); - int kb8_1056 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1054 * 128); - float dbp_1057[8]; - float _tmem_load_12[8]; - tmem_ld_x8(&_tmem_load_12[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float _tmem_load_13[8]; - tmem_ld_x8(&_tmem_load_13[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float _tmem_load_14[8]; - tmem_ld_x8(&_tmem_load_14[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float _tmem_load_15[8]; - tmem_ld_x8(&_tmem_load_15[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float _tmem_load_16[8]; - tmem_ld_x8(&_tmem_load_16[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_1058[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[0])), "=r"(*reinterpret_cast(&btv_1058[(0) + 1])), "=r"(*reinterpret_cast(&btv_1058[(0) + 2])), "=r"(*reinterpret_cast(&btv_1058[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[4])), "=r"(*reinterpret_cast(&btv_1058[(4) + 1])), "=r"(*reinterpret_cast(&btv_1058[(4) + 2])), "=r"(*reinterpret_cast(&btv_1058[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1059[1]; - s_run_1059[0] = 0.0f; - int t_0 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew_1052 + 896 + d_lane]; - float dkf_0 = pool_f32[ew_1052 + 1920 + d_lane]; - float dgf_0 = pool_f32[ew_1052 + 2944 + d_lane]; - float bt_t_0 = btv_1058[7]; - float e_off_0 = approx_exp2(g_t_0 - gn_1049); - float e_dg_0 = approx_exp2(g_t_0 - gc_1050); - float dq2_0 = __fmaf_rn(_tmem_load_12[7], e_off_0, _tmem_load_14[7] * e_dg_0); - float dk2_0 = __fmaf_rn(_tmem_load_13[7], e_off_0, _tmem_load_15[7] * e_dg_0); - dbp_1057[7] = dk2_0 * k_t_0; - float dk2b_0 = dk2_0 * bt_t_0; - float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; - s_run_1059[0] = s_run_1059[0] + dg_o_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_0 = t_0 * qrs; - if (t_0 < clen) { - { - auto _gst_241 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_0; - } - { - auto _gst_242 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_0; - } - } - int t_1 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew_1052 + 768 + d_lane]; - float dkf_1 = pool_f32[ew_1052 + 1792 + d_lane]; - float dgf_1 = pool_f32[ew_1052 + 2816 + d_lane]; - float bt_t_1 = btv_1058[6]; - float e_off_1 = approx_exp2(g_t_1 - gn_1049); - float e_dg_1 = approx_exp2(g_t_1 - gc_1050); - float dq2_1 = __fmaf_rn(_tmem_load_12[6], e_off_1, _tmem_load_14[6] * e_dg_1); - float dk2_1 = __fmaf_rn(_tmem_load_13[6], e_off_1, _tmem_load_15[6] * e_dg_1); - dbp_1057[6] = dk2_1 * k_t_1; - float dk2b_1 = dk2_1 * bt_t_1; - float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + _tmem_load_16[6]; - float dg_o_1 = __fmaf_rn(q_t_1_9, dq2_1, (dk2b_1 - _tmem_load_16[6]) * k_t_1) + dgf_1; - s_run_1059[0] = s_run_1059[0] + dg_o_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1 = t_1 * qrs; - if (t_1 < clen) { - { - auto _gst_243 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_1; - } - { - auto _gst_244 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_1; - } - } - int t_2 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew_1052 + 640 + d_lane]; - float dkf_2 = pool_f32[ew_1052 + 1664 + d_lane]; - float dgf_2 = pool_f32[ew_1052 + 2688 + d_lane]; - float bt_t_2 = btv_1058[5]; - float e_off_2 = approx_exp2(g_t_2 - gn_1049); - float e_dg_2 = approx_exp2(g_t_2 - gc_1050); - float dq2_2 = __fmaf_rn(_tmem_load_12[5], e_off_2, _tmem_load_14[5] * e_dg_2); - float dk2_2 = __fmaf_rn(_tmem_load_13[5], e_off_2, _tmem_load_15[5] * e_dg_2); - dbp_1057[5] = dk2_2 * k_t_2; - float dk2b_2 = dk2_2 * bt_t_2; - float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[5]; - float dg_o_2 = __fmaf_rn(q_t_1_10, dq2_2, (dk2b_2 - _tmem_load_16[5]) * k_t_2) + dgf_2; - s_run_1059[0] = s_run_1059[0] + dg_o_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_2 = t_2 * qrs; - if (t_2 < clen) { - { - auto _gst_245 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_2; - } - { - auto _gst_246 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_2; - } - } - int t_3 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew_1052 + 512 + d_lane]; - float dkf_3 = pool_f32[ew_1052 + 1536 + d_lane]; - float dgf_3 = pool_f32[ew_1052 + 2560 + d_lane]; - float bt_t_3 = btv_1058[4]; - float e_off_3 = approx_exp2(g_t_3 - gn_1049); - float e_dg_3 = approx_exp2(g_t_3 - gc_1050); - float dq2_3 = __fmaf_rn(_tmem_load_12[4], e_off_3, _tmem_load_14[4] * e_dg_3); - float dk2_3 = __fmaf_rn(_tmem_load_13[4], e_off_3, _tmem_load_15[4] * e_dg_3); - dbp_1057[4] = dk2_3 * k_t_3; - float dk2b_3 = dk2_3 * bt_t_3; - float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[4]; - float dg_o_3 = __fmaf_rn(q_t_1_11, dq2_3, (dk2b_3 - _tmem_load_16[4]) * k_t_3) + dgf_3; - s_run_1059[0] = s_run_1059[0] + dg_o_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_3 = t_3 * qrs; - if (t_3 < clen) { - { - auto _gst_247 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_3; - } - { - auto _gst_248 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_3; - } - } - int t_4 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew_1052 + 384 + d_lane]; - float dkf_4 = pool_f32[ew_1052 + 1408 + d_lane]; - float dgf_4 = pool_f32[ew_1052 + 2432 + d_lane]; - float bt_t_4 = btv_1058[3]; - float e_off_4 = approx_exp2(g_t_4 - gn_1049); - float e_dg_4 = approx_exp2(g_t_4 - gc_1050); - float dq2_4 = __fmaf_rn(_tmem_load_12[3], e_off_4, _tmem_load_14[3] * e_dg_4); - float dk2_4 = __fmaf_rn(_tmem_load_13[3], e_off_4, _tmem_load_15[3] * e_dg_4); - dbp_1057[3] = dk2_4 * k_t_4; - float dk2b_4 = dk2_4 * bt_t_4; - float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[3]; - float dg_o_4 = __fmaf_rn(q_t_1_12, dq2_4, (dk2b_4 - _tmem_load_16[3]) * k_t_4) + dgf_4; - s_run_1059[0] = s_run_1059[0] + dg_o_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_4 = t_4 * qrs; - if (t_4 < clen) { - { - auto _gst_249 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_4; - } - { - auto _gst_250 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_4; - } - } - int t_5 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew_1052 + 256 + d_lane]; - float dkf_5 = pool_f32[ew_1052 + 1280 + d_lane]; - float dgf_5 = pool_f32[ew_1052 + 2304 + d_lane]; - float bt_t_5 = btv_1058[2]; - float e_off_5 = approx_exp2(g_t_5 - gn_1049); - float e_dg_5 = approx_exp2(g_t_5 - gc_1050); - float dq2_5 = __fmaf_rn(_tmem_load_12[2], e_off_5, _tmem_load_14[2] * e_dg_5); - float dk2_5 = __fmaf_rn(_tmem_load_13[2], e_off_5, _tmem_load_15[2] * e_dg_5); - dbp_1057[2] = dk2_5 * k_t_5; - float dk2b_5 = dk2_5 * bt_t_5; - float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[2]; - float dg_o_5 = __fmaf_rn(q_t_1_13, dq2_5, (dk2b_5 - _tmem_load_16[2]) * k_t_5) + dgf_5; - s_run_1059[0] = s_run_1059[0] + dg_o_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_5 = t_5 * qrs; - if (t_5 < clen) { - { - auto _gst_251 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_5; - } - { - auto _gst_252 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_5; - } - } - int t_6 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew_1052 + 128 + d_lane]; - float dkf_6 = pool_f32[ew_1052 + 1152 + d_lane]; - float dgf_6 = pool_f32[ew_1052 + 2176 + d_lane]; - float bt_t_6 = btv_1058[1]; - float e_off_6 = approx_exp2(g_t_6 - gn_1049); - float e_dg_6 = approx_exp2(g_t_6 - gc_1050); - float dq2_6 = __fmaf_rn(_tmem_load_12[1], e_off_6, _tmem_load_14[1] * e_dg_6); - float dk2_6 = __fmaf_rn(_tmem_load_13[1], e_off_6, _tmem_load_15[1] * e_dg_6); - dbp_1057[1] = dk2_6 * k_t_6; - float dk2b_6 = dk2_6 * bt_t_6; - float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[1]; - float dg_o_6 = __fmaf_rn(q_t_1_14, dq2_6, (dk2b_6 - _tmem_load_16[1]) * k_t_6) + dgf_6; - s_run_1059[0] = s_run_1059[0] + dg_o_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_6 = t_6 * qrs; - if (t_6 < clen) { - { - auto _gst_253 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_6; - } - { - auto _gst_254 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_6; - } - } - int t_7 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew_1052 + d_lane]; - float dkf_7 = pool_f32[ew_1052 + 1024 + d_lane]; - float dgf_7 = pool_f32[ew_1052 + 2048 + d_lane]; - float bt_t_7 = btv_1058[0]; - float e_off_7 = approx_exp2(g_t_7 - gn_1049); - float e_dg_7 = approx_exp2(g_t_7 - gc_1050); - float dq2_7 = __fmaf_rn(_tmem_load_12[0], e_off_7, _tmem_load_14[0] * e_dg_7); - float dk2_7 = __fmaf_rn(_tmem_load_13[0], e_off_7, _tmem_load_15[0] * e_dg_7); - dbp_1057[0] = dk2_7 * k_t_7; - float dk2b_7 = dk2_7 * bt_t_7; - float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[0]; - float dg_o_7 = __fmaf_rn(q_t_1_15, dq2_7, (dk2b_7 - _tmem_load_16[0]) * k_t_7) + dgf_7; - s_run_1059[0] = s_run_1059[0] + dg_o_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_7 = t_7 * qrs; - if (t_7 < clen) { - { - auto _gst_255 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_7; - } - { - auto _gst_256 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_7; + float r_251[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_0 = r_251[0]; + float gc_1 = gn_0 - 21.640425f; + int eb_0 = 98304 + th * -32768; + int ew_0 = eb_0 >> 2; + int qbase_1 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); + int sb8_0 = 48 + 8 * th; + int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); + int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); + int c_qa_0 = 160 + 8 * th; + int c_ka_0 = 176 + 8 * th; + int c_qd_0 = 224 + 8 * th; + int c_kd_0 = 240 + 8 * th; + int c_dk_0 = 48 + 8 * th; + int c_bp_0 = c_ka_0; + float s_run_0[1]; + s_run_0[0] = 0.0f; + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_1 = 7 - k_1; + float a_qd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1))); + float a_kd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_0 + j_1))); + float a_dk_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_0 + j_1))); + float a_qa_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa_0 + j_1))); + float a_ka_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka_0 + j_1))); + float btr_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_0 + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_0 + j_1; + float r_252[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_252[0]; + unsigned int r_253[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_253[0] << 16); + unsigned int r_254[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1 = __uint_as_float(r_254[0] << 16); + float q_m_1 = ((t_1 >= clen) ? 0.0f : q_t_1); + float dqf_1 = pool_f32[ew_0 + 128 * j_1 + d_lane]; + float dkf_1 = pool_f32[ew_0 + (1024 + 128 * j_1) + d_lane]; + float dgf_1 = pool_f32[ew_0 + (2048 + 128 * j_1) + d_lane]; + float bt_t_1 = btr_1[0]; + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float e_off_1 = approx_exp2(g_t_1 - gn_0); + float dq2_1 = __fmaf_rn(a_qa_1[0], e_off_1, a_qd_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(a_ka_1[0], e_off_1, a_kd_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + a_dk_1[0]; + float qe_1 = q_m_1 * e_dg_1; + float kp_12 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_12; + float kbe_1 = (float)kb_1 * e_dg_1; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, a_qd_1[0], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_12 * e_dg_1) * a_kd_1[0]); + float dg_o_1 = __fmaf_rn(q_m_1, dq2_1, __fmaf_rn(dk2b_1 - a_dk_1[0], k_t_1, cr_1)) + dgf_1; + s_run_0[0] = s_run_0[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_1)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff_1 = t_1 * qrs; + if (t_1 < clen) { + { + auto _gst_259 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_1; + } + { + auto _gst_260 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_1; + } } } - float tot_1228 = s_run_1059[0]; - int b4_1229 = lane_1 >> 4 & 1; - int b3_1230 = lane_1 >> 3 & 1; - int b2_1231 = lane_1 >> 2 & 1; - float v4_1232[4]; - float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_257; - float send_1 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_258; - float send_2 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_259; - float send_3 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_260; - float v2_1237[2]; - float send_4 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_261; - float send_5 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_262; - float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_265; + float tot_0 = s_run_0[0]; + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int b4_0 = lane_1 >> 4 & 1; + int b3_0 = lane_1 >> 3 & 1; + int b2_0 = lane_1 >> 2 & 1; + float v4_0[4]; + float send_4 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_261; + float send_5 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_5, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_262; + float send_6 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_263; + float send_7 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_264; + float v2_16[2]; + float send2_2 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send2_2, 8); + v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_265; + float send2_3 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send2_3, 8); + v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_266; + float send1_0 = ((b2_0 == 0) ? v2_16[1] : v2_16[0]); + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send1_0, 4); + float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_267; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_268; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_269; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); float gacc[8]; @@ -3803,8 +3388,8 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_1228) : "memory"); + uint32_t _addr_270 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_270), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3822,8 +3407,8 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi = ((th == 1) ? tot_1228 : t_other[0]); - float t_lo = ((th == 1) ? t_other[0] : tot_1228); + float t_hi = ((th == 1) ? tot_0 : t_other[0]); + float t_lo = ((th == 1) ? t_other[0] : tot_0); float offset = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi); float sfx[8]; tmem_ld_x8(&sfx[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); @@ -3836,132 +3421,132 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_267 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_271 = make_float2(gacc[4], gacc[4]); float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&rp_u)); - float _abs_270; - asm("abs.f32 %0, %1;" : "=f"(_abs_270) : "f"(rp_z.x)); - float _neg_269; - asm("neg.f32 %0, %1;" : "=f"(_neg_269) : "f"(_abs_270)); - float _exp_268 = __nv_expf(_neg_269); - float _abs_273; - asm("abs.f32 %0, %1;" : "=f"(_abs_273) : "f"(rp_z.y)); - float _neg_272; - asm("neg.f32 %0, %1;" : "=f"(_neg_272) : "f"(_abs_273)); - float _exp_271 = __nv_expf(_neg_272); - float2 rp_e = make_float2(_exp_268, _exp_271); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_271), "l"(*(const unsigned long long*)&rp_u)); + float _abs_274; + asm("abs.f32 %0, %1;" : "=f"(_abs_274) : "f"(rp_z.x)); + float _neg_273; + asm("neg.f32 %0, %1;" : "=f"(_neg_273) : "f"(_abs_274)); + float _exp_272 = __nv_expf(_neg_273); + float _abs_277; + asm("abs.f32 %0, %1;" : "=f"(_abs_277) : "f"(rp_z.y)); + float _neg_276; + asm("neg.f32 %0, %1;" : "=f"(_neg_276) : "f"(_abs_277)); + float _exp_275 = __nv_expf(_neg_276); + float2 rp_e = make_float2(_exp_272, _exp_275); float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_274 = approx_rcp(rp_den.x); - float _rcp_275 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_276 = make_float2(_rcp_274, _rcp_275); + float _rcp_278 = approx_rcp(rp_den.x); + float _rcp_279 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_280 = make_float2(_rcp_278, _rcp_279); float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_276)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_280)); float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_277 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_281 = make_float2(gacc[5], gacc[5]); float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_281)); u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_278 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_282 = make_float2(gacc[4], gacc[4]); float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_278), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_281; - asm("abs.f32 %0, %1;" : "=f"(_abs_281) : "f"(rp_z_0.x)); - float _neg_280; - asm("neg.f32 %0, %1;" : "=f"(_neg_280) : "f"(_abs_281)); - float _exp_279 = __nv_expf(_neg_280); - float _abs_284; - asm("abs.f32 %0, %1;" : "=f"(_abs_284) : "f"(rp_z_0.y)); - float _neg_283; - asm("neg.f32 %0, %1;" : "=f"(_neg_283) : "f"(_abs_284)); - float _exp_282 = __nv_expf(_neg_283); - float2 rp_e_0 = make_float2(_exp_279, _exp_282); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_282), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_285; + asm("abs.f32 %0, %1;" : "=f"(_abs_285) : "f"(rp_z_0.x)); + float _neg_284; + asm("neg.f32 %0, %1;" : "=f"(_neg_284) : "f"(_abs_285)); + float _exp_283 = __nv_expf(_neg_284); + float _abs_288; + asm("abs.f32 %0, %1;" : "=f"(_abs_288) : "f"(rp_z_0.y)); + float _neg_287; + asm("neg.f32 %0, %1;" : "=f"(_neg_287) : "f"(_abs_288)); + float _exp_286 = __nv_expf(_neg_287); + float2 rp_e_0 = make_float2(_exp_283, _exp_286); float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_285 = approx_rcp(rp_den_0.x); - float _rcp_286 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_287 = make_float2(_rcp_285, _rcp_286); + float _rcp_289 = approx_rcp(rp_den_0.x); + float _rcp_290 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_291 = make_float2(_rcp_289, _rcp_290); float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_287)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_291)); float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_288 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_292 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_288)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_292)); u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_289 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_293 = make_float2(gacc[4], gacc[4]); float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_289), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_292; - asm("abs.f32 %0, %1;" : "=f"(_abs_292) : "f"(rp_z_1.x)); - float _neg_291; - asm("neg.f32 %0, %1;" : "=f"(_neg_291) : "f"(_abs_292)); - float _exp_290 = __nv_expf(_neg_291); - float _abs_295; - asm("abs.f32 %0, %1;" : "=f"(_abs_295) : "f"(rp_z_1.y)); - float _neg_294; - asm("neg.f32 %0, %1;" : "=f"(_neg_294) : "f"(_abs_295)); - float _exp_293 = __nv_expf(_neg_294); - float2 rp_e_1 = make_float2(_exp_290, _exp_293); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_293), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_296; + asm("abs.f32 %0, %1;" : "=f"(_abs_296) : "f"(rp_z_1.x)); + float _neg_295; + asm("neg.f32 %0, %1;" : "=f"(_neg_295) : "f"(_abs_296)); + float _exp_294 = __nv_expf(_neg_295); + float _abs_299; + asm("abs.f32 %0, %1;" : "=f"(_abs_299) : "f"(rp_z_1.y)); + float _neg_298; + asm("neg.f32 %0, %1;" : "=f"(_neg_298) : "f"(_abs_299)); + float _exp_297 = __nv_expf(_neg_298); + float2 rp_e_1 = make_float2(_exp_294, _exp_297); float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_296 = approx_rcp(rp_den_1.x); - float _rcp_297 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_298 = make_float2(_rcp_296, _rcp_297); + float _rcp_300 = approx_rcp(rp_den_1.x); + float _rcp_301 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_302 = make_float2(_rcp_300, _rcp_301); float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_298)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_302)); float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_299 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_303 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_299)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_303)); u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_300 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_304 = make_float2(gacc[4], gacc[4]); float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_303; - asm("abs.f32 %0, %1;" : "=f"(_abs_303) : "f"(rp_z_2.x)); - float _neg_302; - asm("neg.f32 %0, %1;" : "=f"(_neg_302) : "f"(_abs_303)); - float _exp_301 = __nv_expf(_neg_302); - float _abs_306; - asm("abs.f32 %0, %1;" : "=f"(_abs_306) : "f"(rp_z_2.y)); - float _neg_305; - asm("neg.f32 %0, %1;" : "=f"(_neg_305) : "f"(_abs_306)); - float _exp_304 = __nv_expf(_neg_305); - float2 rp_e_2 = make_float2(_exp_301, _exp_304); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_304), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_307; + asm("abs.f32 %0, %1;" : "=f"(_abs_307) : "f"(rp_z_2.x)); + float _neg_306; + asm("neg.f32 %0, %1;" : "=f"(_neg_306) : "f"(_abs_307)); + float _exp_305 = __nv_expf(_neg_306); + float _abs_310; + asm("abs.f32 %0, %1;" : "=f"(_abs_310) : "f"(rp_z_2.y)); + float _neg_309; + asm("neg.f32 %0, %1;" : "=f"(_neg_309) : "f"(_abs_310)); + float _exp_308 = __nv_expf(_neg_309); + float2 rp_e_2 = make_float2(_exp_305, _exp_308); float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_307 = approx_rcp(rp_den_2.x); - float _rcp_308 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_309 = make_float2(_rcp_307, _rcp_308); + float _rcp_311 = approx_rcp(rp_den_2.x); + float _rcp_312 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_313 = make_float2(_rcp_311, _rcp_312); float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_313)); float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_310 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_314 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_310)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_314)); u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -3976,54 +3561,54 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_311; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_311) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_311; + uint32_t _prmt_315; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_315; int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_312 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = pq_0[0]; + auto _gst_316 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = pq_0[0]; } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_313; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_313) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_313; + uint32_t _prmt_317; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_317; int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_314 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = pq_1[0]; + auto _gst_318 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = pq_1[0]; } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_315; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_315; + uint32_t _prmt_319; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_319) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_319; int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_316 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = pq_2[0]; + auto _gst_320 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = pq_2[0]; } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_317; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_317; + uint32_t _prmt_321; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_321) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_321; int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_318 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = pq_3[0]; + auto _gst_322 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast(_gst_322) + (0)) = pq_3[0]; } } int t_a_0 = 48 + 8 * th; @@ -4088,47 +3673,47 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_319; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_319) : "f"(s_a), "f"(nm_t)); + float _sub_323; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_323) : "f"(s_a), "f"(nm_t)); float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_319), "f"(pa)); - float _sub_320; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_320) : "f"(pa), "f"(nm_t)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_323), "f"(pa)); + float _sub_324; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_324) : "f"(pa), "f"(nm_t)); float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_320), "f"(s_a)); - float _abs_322; - asm("abs.f32 %0, %1;" : "=f"(_abs_322) : "f"(s_a)); - float _abs_323; - asm("abs.f32 %0, %1;" : "=f"(_abs_323) : "f"(pa)); - uint32_t _setp_321_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_321_bits) : "f"(_abs_322), "f"(_abs_323)); - bool _setp_321 = _setp_321_bits != 0; - float nm_err = ((_setp_321) ? nm_es : nm_ex); - float _add_324; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(c_a), "f"(nm_err)); - c_a = _add_324; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_324), "f"(s_a)); + float _abs_326; + asm("abs.f32 %0, %1;" : "=f"(_abs_326) : "f"(s_a)); + float _abs_327; + asm("abs.f32 %0, %1;" : "=f"(_abs_327) : "f"(pa)); + uint32_t _setp_325_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_325_bits) : "f"(_abs_326), "f"(_abs_327)); + bool _setp_325 = _setp_325_bits != 0; + float nm_err = ((_setp_325) ? nm_es : nm_ex); + float _add_328; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(c_a), "f"(nm_err)); + c_a = _add_328; s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_325; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_325) : "f"(s_b), "f"(nm_t_0)); + float _sub_329; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_329) : "f"(s_b), "f"(nm_t_0)); float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_325), "f"(pb)); - float _sub_326; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_326) : "f"(pb), "f"(nm_t_0)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_329), "f"(pb)); + float _sub_330; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_330) : "f"(pb), "f"(nm_t_0)); float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_326), "f"(s_b)); - float _abs_328; - asm("abs.f32 %0, %1;" : "=f"(_abs_328) : "f"(s_b)); - float _abs_329; - asm("abs.f32 %0, %1;" : "=f"(_abs_329) : "f"(pb)); - uint32_t _setp_327_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_327_bits) : "f"(_abs_328), "f"(_abs_329)); - bool _setp_327 = _setp_327_bits != 0; - float nm_err_0 = ((_setp_327) ? nm_es_0 : nm_ex_0); - float _add_330; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_330; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_330), "f"(s_b)); + float _abs_332; + asm("abs.f32 %0, %1;" : "=f"(_abs_332) : "f"(s_b)); + float _abs_333; + asm("abs.f32 %0, %1;" : "=f"(_abs_333) : "f"(pb)); + uint32_t _setp_331_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_331_bits) : "f"(_abs_332), "f"(_abs_333)); + bool _setp_331 = _setp_331_bits != 0; + float nm_err_0 = ((_setp_331) ? nm_es_0 : nm_ex_0); + float _add_334; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_334; s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4152,132 +3737,132 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_331 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_335 = make_float2(gacc[4], gacc[4]); float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_331), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_334; - asm("abs.f32 %0, %1;" : "=f"(_abs_334) : "f"(rp_z_3.x)); - float _neg_333; - asm("neg.f32 %0, %1;" : "=f"(_neg_333) : "f"(_abs_334)); - float _exp_332 = __nv_expf(_neg_333); - float _abs_337; - asm("abs.f32 %0, %1;" : "=f"(_abs_337) : "f"(rp_z_3.y)); - float _neg_336; - asm("neg.f32 %0, %1;" : "=f"(_neg_336) : "f"(_abs_337)); - float _exp_335 = __nv_expf(_neg_336); - float2 rp_e_3 = make_float2(_exp_332, _exp_335); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_335), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_338; + asm("abs.f32 %0, %1;" : "=f"(_abs_338) : "f"(rp_z_3.x)); + float _neg_337; + asm("neg.f32 %0, %1;" : "=f"(_neg_337) : "f"(_abs_338)); + float _exp_336 = __nv_expf(_neg_337); + float _abs_341; + asm("abs.f32 %0, %1;" : "=f"(_abs_341) : "f"(rp_z_3.y)); + float _neg_340; + asm("neg.f32 %0, %1;" : "=f"(_neg_340) : "f"(_abs_341)); + float _exp_339 = __nv_expf(_neg_340); + float2 rp_e_3 = make_float2(_exp_336, _exp_339); float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_338 = approx_rcp(rp_den_3.x); - float _rcp_339 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_340 = make_float2(_rcp_338, _rcp_339); + float _rcp_342 = approx_rcp(rp_den_3.x); + float _rcp_343 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_344 = make_float2(_rcp_342, _rcp_343); float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_340)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_344)); float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_341 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_345 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_345)); u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_342 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_346 = make_float2(gacc[4], gacc[4]); float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_342), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_345; - asm("abs.f32 %0, %1;" : "=f"(_abs_345) : "f"(rp_z_4.x)); - float _neg_344; - asm("neg.f32 %0, %1;" : "=f"(_neg_344) : "f"(_abs_345)); - float _exp_343 = __nv_expf(_neg_344); - float _abs_348; - asm("abs.f32 %0, %1;" : "=f"(_abs_348) : "f"(rp_z_4.y)); - float _neg_347; - asm("neg.f32 %0, %1;" : "=f"(_neg_347) : "f"(_abs_348)); - float _exp_346 = __nv_expf(_neg_347); - float2 rp_e_4 = make_float2(_exp_343, _exp_346); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_346), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_349; + asm("abs.f32 %0, %1;" : "=f"(_abs_349) : "f"(rp_z_4.x)); + float _neg_348; + asm("neg.f32 %0, %1;" : "=f"(_neg_348) : "f"(_abs_349)); + float _exp_347 = __nv_expf(_neg_348); + float _abs_352; + asm("abs.f32 %0, %1;" : "=f"(_abs_352) : "f"(rp_z_4.y)); + float _neg_351; + asm("neg.f32 %0, %1;" : "=f"(_neg_351) : "f"(_abs_352)); + float _exp_350 = __nv_expf(_neg_351); + float2 rp_e_4 = make_float2(_exp_347, _exp_350); float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_349 = approx_rcp(rp_den_4.x); - float _rcp_350 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_351 = make_float2(_rcp_349, _rcp_350); + float _rcp_353 = approx_rcp(rp_den_4.x); + float _rcp_354 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_355 = make_float2(_rcp_353, _rcp_354); float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_351)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_355)); float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_352 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_356 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_352)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_356)); u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_353 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_357 = make_float2(gacc[4], gacc[4]); float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_356; - asm("abs.f32 %0, %1;" : "=f"(_abs_356) : "f"(rp_z_5.x)); - float _neg_355; - asm("neg.f32 %0, %1;" : "=f"(_neg_355) : "f"(_abs_356)); - float _exp_354 = __nv_expf(_neg_355); - float _abs_359; - asm("abs.f32 %0, %1;" : "=f"(_abs_359) : "f"(rp_z_5.y)); - float _neg_358; - asm("neg.f32 %0, %1;" : "=f"(_neg_358) : "f"(_abs_359)); - float _exp_357 = __nv_expf(_neg_358); - float2 rp_e_5 = make_float2(_exp_354, _exp_357); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_357), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_360; + asm("abs.f32 %0, %1;" : "=f"(_abs_360) : "f"(rp_z_5.x)); + float _neg_359; + asm("neg.f32 %0, %1;" : "=f"(_neg_359) : "f"(_abs_360)); + float _exp_358 = __nv_expf(_neg_359); + float _abs_363; + asm("abs.f32 %0, %1;" : "=f"(_abs_363) : "f"(rp_z_5.y)); + float _neg_362; + asm("neg.f32 %0, %1;" : "=f"(_neg_362) : "f"(_abs_363)); + float _exp_361 = __nv_expf(_neg_362); + float2 rp_e_5 = make_float2(_exp_358, _exp_361); float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_360 = approx_rcp(rp_den_5.x); - float _rcp_361 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_362 = make_float2(_rcp_360, _rcp_361); + float _rcp_364 = approx_rcp(rp_den_5.x); + float _rcp_365 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_366 = make_float2(_rcp_364, _rcp_365); float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_362)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_366)); float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_363 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_367 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_363)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_367)); u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_364 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_368 = make_float2(gacc[4], gacc[4]); float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_364), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_367; - asm("abs.f32 %0, %1;" : "=f"(_abs_367) : "f"(rp_z_6.x)); - float _neg_366; - asm("neg.f32 %0, %1;" : "=f"(_neg_366) : "f"(_abs_367)); - float _exp_365 = __nv_expf(_neg_366); - float _abs_370; - asm("abs.f32 %0, %1;" : "=f"(_abs_370) : "f"(rp_z_6.y)); - float _neg_369; - asm("neg.f32 %0, %1;" : "=f"(_neg_369) : "f"(_abs_370)); - float _exp_368 = __nv_expf(_neg_369); - float2 rp_e_6 = make_float2(_exp_365, _exp_368); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_368), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_371; + asm("abs.f32 %0, %1;" : "=f"(_abs_371) : "f"(rp_z_6.x)); + float _neg_370; + asm("neg.f32 %0, %1;" : "=f"(_neg_370) : "f"(_abs_371)); + float _exp_369 = __nv_expf(_neg_370); + float _abs_374; + asm("abs.f32 %0, %1;" : "=f"(_abs_374) : "f"(rp_z_6.y)); + float _neg_373; + asm("neg.f32 %0, %1;" : "=f"(_neg_373) : "f"(_abs_374)); + float _exp_372 = __nv_expf(_neg_373); + float2 rp_e_6 = make_float2(_exp_369, _exp_372); float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_371 = approx_rcp(rp_den_6.x); - float _rcp_372 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_373 = make_float2(_rcp_371, _rcp_372); + float _rcp_375 = approx_rcp(rp_den_6.x); + float _rcp_376 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_377 = make_float2(_rcp_375, _rcp_376); float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_373)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_377)); float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_374 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_378 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_374)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_378)); u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4292,54 +3877,54 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_375; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_375) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_375; + uint32_t _prmt_379; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_379; int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_376 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_376)); - *(reinterpret_cast(_gst_376) + (0)) = pq_4[0]; + auto _gst_380 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = pq_4[0]; } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_377; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_377) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_377; + uint32_t _prmt_381; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_381; int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_378 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_378)); - *(reinterpret_cast(_gst_378) + (0)) = pq_5[0]; + auto _gst_382 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = pq_5[0]; } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_379; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_379; + uint32_t _prmt_383; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_383) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_383; int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_380 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_380)); - *(reinterpret_cast(_gst_380) + (0)) = pq_6[0]; + auto _gst_384 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = pq_6[0]; } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_381; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_381; + uint32_t _prmt_385; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_385) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_385; int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_382 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_382)); - *(reinterpret_cast(_gst_382) + (0)) = pq_7[0]; + auto _gst_386 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_386)); + *(reinterpret_cast(_gst_386) + (0)) = pq_7[0]; } } int t_a_8 = 32 + 8 * th; @@ -4404,47 +3989,47 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_383; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_383) : "f"(s_a_0), "f"(nm_t_1)); + float _sub_387; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_387) : "f"(s_a_0), "f"(nm_t_1)); float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_383), "f"(pa_0)); - float _sub_384; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_384) : "f"(pa_0), "f"(nm_t_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_387), "f"(pa_0)); + float _sub_388; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_388) : "f"(pa_0), "f"(nm_t_1)); float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_384), "f"(s_a_0)); - float _abs_386; - asm("abs.f32 %0, %1;" : "=f"(_abs_386) : "f"(s_a_0)); - float _abs_387; - asm("abs.f32 %0, %1;" : "=f"(_abs_387) : "f"(pa_0)); - uint32_t _setp_385_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_385_bits) : "f"(_abs_386), "f"(_abs_387)); - bool _setp_385 = _setp_385_bits != 0; - float nm_err_1 = ((_setp_385) ? nm_es_1 : nm_ex_1); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_388), "f"(s_a_0)); + float _abs_390; + asm("abs.f32 %0, %1;" : "=f"(_abs_390) : "f"(s_a_0)); + float _abs_391; + asm("abs.f32 %0, %1;" : "=f"(_abs_391) : "f"(pa_0)); + uint32_t _setp_389_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_389_bits) : "f"(_abs_390), "f"(_abs_391)); + bool _setp_389 = _setp_389_bits != 0; + float nm_err_1 = ((_setp_389) ? nm_es_1 : nm_ex_1); + float _add_392; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_392; s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_389; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_389) : "f"(s_b_0), "f"(nm_t_2)); + float _sub_393; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_393) : "f"(s_b_0), "f"(nm_t_2)); float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_389), "f"(pb_0)); - float _sub_390; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_390) : "f"(pb_0), "f"(nm_t_2)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_393), "f"(pb_0)); + float _sub_394; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_394) : "f"(pb_0), "f"(nm_t_2)); float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_390), "f"(s_b_0)); - float _abs_392; - asm("abs.f32 %0, %1;" : "=f"(_abs_392) : "f"(s_b_0)); - float _abs_393; - asm("abs.f32 %0, %1;" : "=f"(_abs_393) : "f"(pb_0)); - uint32_t _setp_391_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_391_bits) : "f"(_abs_392), "f"(_abs_393)); - bool _setp_391 = _setp_391_bits != 0; - float nm_err_2 = ((_setp_391) ? nm_es_2 : nm_ex_2); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_394; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_394), "f"(s_b_0)); + float _abs_396; + asm("abs.f32 %0, %1;" : "=f"(_abs_396) : "f"(s_b_0)); + float _abs_397; + asm("abs.f32 %0, %1;" : "=f"(_abs_397) : "f"(pb_0)); + uint32_t _setp_395_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_395_bits) : "f"(_abs_396), "f"(_abs_397)); + bool _setp_395 = _setp_395_bits != 0; + float nm_err_2 = ((_setp_395) ? nm_es_2 : nm_ex_2); + float _add_398; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_398; s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4452,8 +4037,8 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_395 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_395), "f"(off_acc[0]) : "memory"); + uint32_t _addr_399 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_399), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4479,9 +4064,9 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1292 = r_297[0]; + float r_255[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1292 = r_255[0]; float gc_1293 = gn_1292 - 21.640425f; int eb_1294 = 81920 + th * -32768; int ew_1295 = eb_1294 >> 2; @@ -4510,348 +4095,388 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1302[1]; s_run_1302[0] = 0.0f; - int t_8 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_1295 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_1295 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_1295 + 2944 + d_lane]; - float bt_t_8 = btv_1301[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_1292); - float e_dg_8 = approx_exp2(g_t_8 - gc_1293); - float dq2_8 = __fmaf_rn(_tmem_load_20[7], e_off_8, _tmem_load_22[7] * e_dg_8); - float dk2_8 = __fmaf_rn(_tmem_load_21[7], e_off_8, _tmem_load_23[7] * e_dg_8); - dbp_1300[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_24[7]; - float dg_o_8 = __fmaf_rn(q_t_1_16, dq2_8, (dk2b_8 - _tmem_load_24[7]) * k_t_8) + dgf_8; - s_run_1302[0] = s_run_1302[0] + dg_o_8; + int t_0 = 16 + 8 * th + 7; + float r_256[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_0 = r_256[0]; + unsigned int r_257[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_0 = __uint_as_float(r_257[0] << 16); + unsigned int r_258[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_0 = __uint_as_float(r_258[0] << 16); + float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); + float dqf_0 = pool_f32[ew_1295 + 896 + d_lane]; + float dkf_0 = pool_f32[ew_1295 + 1920 + d_lane]; + float dgf_0 = pool_f32[ew_1295 + 2944 + d_lane]; + float bt_t_0 = btv_1301[7]; + float e_off_0 = approx_exp2(g_t_0 - gn_1292); + float e_dg_0 = approx_exp2(g_t_0 - gc_1293); + float dq2_0 = __fmaf_rn(_tmem_load_20[7], e_off_0, _tmem_load_22[7] * e_dg_0); + float dk2_0 = __fmaf_rn(_tmem_load_21[7], e_off_0, _tmem_load_23[7] * e_dg_0); + dbp_1300[7] = dk2_0 * k_t_0; + float dk2b_0 = dk2_0 * bt_t_0; + float dq_o_0 = dq2_0 + dqf_0; + float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_24[7]; + float qe_2 = q_t_1_0 * e_dg_0; + float kp_13 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_13; + float kbe_2 = (float)kb_2 * e_dg_0; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_13 * e_dg_0) * _tmem_load_23[7]); + float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_24[7], k_t_0, cr_2)) + dgf_0; + s_run_1302[0] = s_run_1302[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_8 = t_8 * qrs; - if (t_8 < clen) { + int qoff_0 = t_0 * qrs; + if (t_0 < clen) { { - auto _gst_396 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_396)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_396) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_400 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_400)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_397 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_397)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_397) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_401 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_401)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_0; } } - int t_9 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_1295 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_1295 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_1295 + 2816 + d_lane]; - float bt_t_9 = btv_1301[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_1292); - float e_dg_9 = approx_exp2(g_t_9 - gc_1293); - float dq2_9 = __fmaf_rn(_tmem_load_20[6], e_off_9, _tmem_load_22[6] * e_dg_9); - float dk2_9 = __fmaf_rn(_tmem_load_21[6], e_off_9, _tmem_load_23[6] * e_dg_9); - dbp_1300[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[6]; - float dg_o_9 = __fmaf_rn(q_t_1_17, dq2_9, (dk2b_9 - _tmem_load_24[6]) * k_t_9) + dgf_9; - s_run_1302[0] = s_run_1302[0] + dg_o_9; + int t_1_1 = 16 + 8 * th + 6; + float r_259[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1_1 = r_259[0]; + unsigned int r_260[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1_1 = __uint_as_float(r_260[0] << 16); + unsigned int r_261[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1_1 = __uint_as_float(r_261[0] << 16); + float q_t_1_1_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_1); + float dqf_1_1 = pool_f32[ew_1295 + 768 + d_lane]; + float dkf_1_1 = pool_f32[ew_1295 + 1792 + d_lane]; + float dgf_1_1 = pool_f32[ew_1295 + 2816 + d_lane]; + float bt_t_1_1 = btv_1301[6]; + float e_off_1_1 = approx_exp2(g_t_1_1 - gn_1292); + float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_1293); + float dq2_1_1 = __fmaf_rn(_tmem_load_20[6], e_off_1_1, _tmem_load_22[6] * e_dg_1_1); + float dk2_1_1 = __fmaf_rn(_tmem_load_21[6], e_off_1_1, _tmem_load_23[6] * e_dg_1_1); + dbp_1300[6] = dk2_1_1 * k_t_1_1; + float dk2b_1_1 = dk2_1_1 * bt_t_1_1; + float dq_o_1_1 = dq2_1_1 + dqf_1_1; + float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_24[6]; + float qe_0 = q_t_1_1_1 * e_dg_1_1; + float kp_12_1 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12_1; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12_1 * e_dg_1_1) * _tmem_load_23[6]); + float dg_o_1_1 = __fmaf_rn(q_t_1_1_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_24[6], k_t_1_1, cr_0)) + dgf_1_1; + s_run_1302[0] = s_run_1302[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_9 = t_9 * qrs; - if (t_9 < clen) { + int qoff_1_1 = t_1_1 * qrs; + if (t_1_1 < clen) { { - auto _gst_398 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_398)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_398) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_402 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_402)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_1_1; } { - auto _gst_399 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_399)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_399) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_403 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_403)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_1_1; } } - int t_10 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_1295 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_1295 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_1295 + 2688 + d_lane]; - float bt_t_10 = btv_1301[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_1292); - float e_dg_10 = approx_exp2(g_t_10 - gc_1293); - float dq2_10 = __fmaf_rn(_tmem_load_20[5], e_off_10, _tmem_load_22[5] * e_dg_10); - float dk2_10 = __fmaf_rn(_tmem_load_21[5], e_off_10, _tmem_load_23[5] * e_dg_10); - dbp_1300[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[5]; - float dg_o_10 = __fmaf_rn(q_t_1_18, dq2_10, (dk2b_10 - _tmem_load_24[5]) * k_t_10) + dgf_10; - s_run_1302[0] = s_run_1302[0] + dg_o_10; + int t_2 = 16 + 8 * th + 5; + float r_262[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_262[0]; + unsigned int r_263[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_263[0] << 16); + unsigned int r_264[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_264[0] << 16); + float q_t_1_2 = ((t_2 >= clen) ? 0.0f : q_t_2); + float dqf_2 = pool_f32[ew_1295 + 640 + d_lane]; + float dkf_2 = pool_f32[ew_1295 + 1664 + d_lane]; + float dgf_2 = pool_f32[ew_1295 + 2688 + d_lane]; + float bt_t_2 = btv_1301[5]; + float e_off_2 = approx_exp2(g_t_2 - gn_1292); + float e_dg_2 = approx_exp2(g_t_2 - gc_1293); + float dq2_2 = __fmaf_rn(_tmem_load_20[5], e_off_2, _tmem_load_22[5] * e_dg_2); + float dk2_2 = __fmaf_rn(_tmem_load_21[5], e_off_2, _tmem_load_23[5] * e_dg_2); + dbp_1300[5] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_24[5]; + float qe_1_1 = q_t_1_2 * e_dg_2; + float kp_13_1 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1_1 = (__nv_bfloat16)kp_13_1; + float kbe_1_1 = (float)kb_1_1 * e_dg_2; + float cr_1_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1_1) & 4294959104u) - qe_1_1, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_1_1) & 4294959104u) - kp_13_1 * e_dg_2) * _tmem_load_23[5]); + float dg_o_2 = __fmaf_rn(q_t_1_2, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_24[5], k_t_2, cr_1_1)) + dgf_2; + s_run_1302[0] = s_run_1302[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_10 = t_10 * qrs; - if (t_10 < clen) { + int qoff_2 = t_2 * qrs; + if (t_2 < clen) { { - auto _gst_400 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_400)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_404 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_404)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_401 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_401)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_405 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_405)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_2; } } - int t_11 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_1295 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_1295 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_1295 + 2560 + d_lane]; - float bt_t_11 = btv_1301[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_1292); - float e_dg_11 = approx_exp2(g_t_11 - gc_1293); - float dq2_11 = __fmaf_rn(_tmem_load_20[4], e_off_11, _tmem_load_22[4] * e_dg_11); - float dk2_11 = __fmaf_rn(_tmem_load_21[4], e_off_11, _tmem_load_23[4] * e_dg_11); - dbp_1300[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[4]; - float dg_o_11 = __fmaf_rn(q_t_1_19, dq2_11, (dk2b_11 - _tmem_load_24[4]) * k_t_11) + dgf_11; - s_run_1302[0] = s_run_1302[0] + dg_o_11; + int t_3 = 16 + 8 * th + 4; + float r_265[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_3 = r_265[0]; + unsigned int r_266[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_3 = __uint_as_float(r_266[0] << 16); + unsigned int r_267[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_3 = __uint_as_float(r_267[0] << 16); + float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); + float dqf_3 = pool_f32[ew_1295 + 512 + d_lane]; + float dkf_3 = pool_f32[ew_1295 + 1536 + d_lane]; + float dgf_3 = pool_f32[ew_1295 + 2560 + d_lane]; + float bt_t_3 = btv_1301[4]; + float e_off_3 = approx_exp2(g_t_3 - gn_1292); + float e_dg_3 = approx_exp2(g_t_3 - gc_1293); + float dq2_3 = __fmaf_rn(_tmem_load_20[4], e_off_3, _tmem_load_22[4] * e_dg_3); + float dk2_3 = __fmaf_rn(_tmem_load_21[4], e_off_3, _tmem_load_23[4] * e_dg_3); + dbp_1300[4] = dk2_3 * k_t_3; + float dk2b_3 = dk2_3 * bt_t_3; + float dq_o_3 = dq2_3 + dqf_3; + float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_24[4]; + float qe_2_1 = q_t_1_3 * e_dg_3; + float kp_14 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2_1 = (__nv_bfloat16)kp_14; + float kbe_2_1 = (float)kb_2_1 * e_dg_3; + float cr_2_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_2_1) & 4294959104u) - qe_2_1, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_2_1) & 4294959104u) - kp_14 * e_dg_3) * _tmem_load_23[4]); + float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_24[4], k_t_3, cr_2_1)) + dgf_3; + s_run_1302[0] = s_run_1302[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_11 = t_11 * qrs; - if (t_11 < clen) { + int qoff_3 = t_3 * qrs; + if (t_3 < clen) { { - auto _gst_402 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_402)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_406 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_406)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_403 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_403)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_407 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_407)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_3; } } - int t_12 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_1295 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_1295 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_1295 + 2432 + d_lane]; - float bt_t_12 = btv_1301[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_1292); - float e_dg_12 = approx_exp2(g_t_12 - gc_1293); - float dq2_12 = __fmaf_rn(_tmem_load_20[3], e_off_12, _tmem_load_22[3] * e_dg_12); - float dk2_12 = __fmaf_rn(_tmem_load_21[3], e_off_12, _tmem_load_23[3] * e_dg_12); - dbp_1300[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[3]; - float dg_o_12 = __fmaf_rn(q_t_1_20, dq2_12, (dk2b_12 - _tmem_load_24[3]) * k_t_12) + dgf_12; - s_run_1302[0] = s_run_1302[0] + dg_o_12; + int t_4 = 16 + 8 * th + 3; + float r_268[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_4 = r_268[0]; + unsigned int r_269[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_4 = __uint_as_float(r_269[0] << 16); + unsigned int r_270[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_4 = __uint_as_float(r_270[0] << 16); + float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); + float dqf_4 = pool_f32[ew_1295 + 384 + d_lane]; + float dkf_4 = pool_f32[ew_1295 + 1408 + d_lane]; + float dgf_4 = pool_f32[ew_1295 + 2432 + d_lane]; + float bt_t_4 = btv_1301[3]; + float e_off_4 = approx_exp2(g_t_4 - gn_1292); + float e_dg_4 = approx_exp2(g_t_4 - gc_1293); + float dq2_4 = __fmaf_rn(_tmem_load_20[3], e_off_4, _tmem_load_22[3] * e_dg_4); + float dk2_4 = __fmaf_rn(_tmem_load_21[3], e_off_4, _tmem_load_23[3] * e_dg_4); + dbp_1300[3] = dk2_4 * k_t_4; + float dk2b_4 = dk2_4 * bt_t_4; + float dq_o_4 = dq2_4 + dqf_4; + float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_24[3]; + float qe_3 = q_t_1_4 * e_dg_4; + float kp_15 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_4) * _tmem_load_23[3]); + float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_24[3], k_t_4, cr_3)) + dgf_4; + s_run_1302[0] = s_run_1302[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_12 = t_12 * qrs; - if (t_12 < clen) { + int qoff_4 = t_4 * qrs; + if (t_4 < clen) { { - auto _gst_404 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_404)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_408 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_408)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_405 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_405)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_409 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_409)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_4; } } - int t_13 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_1295 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_1295 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_1295 + 2304 + d_lane]; - float bt_t_13 = btv_1301[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_1292); - float e_dg_13 = approx_exp2(g_t_13 - gc_1293); - float dq2_13 = __fmaf_rn(_tmem_load_20[2], e_off_13, _tmem_load_22[2] * e_dg_13); - float dk2_13 = __fmaf_rn(_tmem_load_21[2], e_off_13, _tmem_load_23[2] * e_dg_13); - dbp_1300[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[2]; - float dg_o_13 = __fmaf_rn(q_t_1_21, dq2_13, (dk2b_13 - _tmem_load_24[2]) * k_t_13) + dgf_13; - s_run_1302[0] = s_run_1302[0] + dg_o_13; + int t_5 = 16 + 8 * th + 2; + float r_271[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_5 = r_271[0]; + unsigned int r_272[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_5 = __uint_as_float(r_272[0] << 16); + unsigned int r_273[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_5 = __uint_as_float(r_273[0] << 16); + float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); + float dqf_5 = pool_f32[ew_1295 + 256 + d_lane]; + float dkf_5 = pool_f32[ew_1295 + 1280 + d_lane]; + float dgf_5 = pool_f32[ew_1295 + 2304 + d_lane]; + float bt_t_5 = btv_1301[2]; + float e_off_5 = approx_exp2(g_t_5 - gn_1292); + float e_dg_5 = approx_exp2(g_t_5 - gc_1293); + float dq2_5 = __fmaf_rn(_tmem_load_20[2], e_off_5, _tmem_load_22[2] * e_dg_5); + float dk2_5 = __fmaf_rn(_tmem_load_21[2], e_off_5, _tmem_load_23[2] * e_dg_5); + dbp_1300[2] = dk2_5 * k_t_5; + float dk2b_5 = dk2_5 * bt_t_5; + float dq_o_5 = dq2_5 + dqf_5; + float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_24[2]; + float qe_4 = q_t_1_5 * e_dg_5; + float kp_16 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_5) * _tmem_load_23[2]); + float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_24[2], k_t_5, cr_4)) + dgf_5; + s_run_1302[0] = s_run_1302[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_13 = t_13 * qrs; - if (t_13 < clen) { + int qoff_5 = t_5 * qrs; + if (t_5 < clen) { { - auto _gst_406 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_406)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_410 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_410)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_407 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_407)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_411 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_411)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_5; } } - int t_14 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_1295 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_1295 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_1295 + 2176 + d_lane]; - float bt_t_14 = btv_1301[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_1292); - float e_dg_14 = approx_exp2(g_t_14 - gc_1293); - float dq2_14 = __fmaf_rn(_tmem_load_20[1], e_off_14, _tmem_load_22[1] * e_dg_14); - float dk2_14 = __fmaf_rn(_tmem_load_21[1], e_off_14, _tmem_load_23[1] * e_dg_14); - dbp_1300[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[1]; - float dg_o_14 = __fmaf_rn(q_t_1_22, dq2_14, (dk2b_14 - _tmem_load_24[1]) * k_t_14) + dgf_14; - s_run_1302[0] = s_run_1302[0] + dg_o_14; + int t_6 = 16 + 8 * th + 1; + float r_274[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_6 = r_274[0]; + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_6 = __uint_as_float(r_275[0] << 16); + unsigned int r_276[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_6 = __uint_as_float(r_276[0] << 16); + float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); + float dqf_6 = pool_f32[ew_1295 + 128 + d_lane]; + float dkf_6 = pool_f32[ew_1295 + 1152 + d_lane]; + float dgf_6 = pool_f32[ew_1295 + 2176 + d_lane]; + float bt_t_6 = btv_1301[1]; + float e_off_6 = approx_exp2(g_t_6 - gn_1292); + float e_dg_6 = approx_exp2(g_t_6 - gc_1293); + float dq2_6 = __fmaf_rn(_tmem_load_20[1], e_off_6, _tmem_load_22[1] * e_dg_6); + float dk2_6 = __fmaf_rn(_tmem_load_21[1], e_off_6, _tmem_load_23[1] * e_dg_6); + dbp_1300[1] = dk2_6 * k_t_6; + float dk2b_6 = dk2_6 * bt_t_6; + float dq_o_6 = dq2_6 + dqf_6; + float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_24[1]; + float qe_5 = q_t_1_6 * e_dg_6; + float kp_17 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_6) * _tmem_load_23[1]); + float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_24[1], k_t_6, cr_5)) + dgf_6; + s_run_1302[0] = s_run_1302[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_14 = t_14 * qrs; - if (t_14 < clen) { + int qoff_6 = t_6 * qrs; + if (t_6 < clen) { { - auto _gst_408 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_408)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_412 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_412)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_412) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_409 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_409)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_413 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_413)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_413) + (0)) = (__nv_bfloat16)dk_o_6; } } - int t_15 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_1295 + d_lane]; - float dkf_15 = pool_f32[ew_1295 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_1295 + 2048 + d_lane]; - float bt_t_15 = btv_1301[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_1292); - float e_dg_15 = approx_exp2(g_t_15 - gc_1293); - float dq2_15 = __fmaf_rn(_tmem_load_20[0], e_off_15, _tmem_load_22[0] * e_dg_15); - float dk2_15 = __fmaf_rn(_tmem_load_21[0], e_off_15, _tmem_load_23[0] * e_dg_15); - dbp_1300[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[0]; - float dg_o_15 = __fmaf_rn(q_t_1_23, dq2_15, (dk2b_15 - _tmem_load_24[0]) * k_t_15) + dgf_15; - s_run_1302[0] = s_run_1302[0] + dg_o_15; + int t_7 = 16 + 8 * th; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_7 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_7 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_7 = __uint_as_float(r_279[0] << 16); + float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); + float dqf_7 = pool_f32[ew_1295 + d_lane]; + float dkf_7 = pool_f32[ew_1295 + 1024 + d_lane]; + float dgf_7 = pool_f32[ew_1295 + 2048 + d_lane]; + float bt_t_7 = btv_1301[0]; + float e_off_7 = approx_exp2(g_t_7 - gn_1292); + float e_dg_7 = approx_exp2(g_t_7 - gc_1293); + float dq2_7 = __fmaf_rn(_tmem_load_20[0], e_off_7, _tmem_load_22[0] * e_dg_7); + float dk2_7 = __fmaf_rn(_tmem_load_21[0], e_off_7, _tmem_load_23[0] * e_dg_7); + dbp_1300[0] = dk2_7 * k_t_7; + float dk2b_7 = dk2_7 * bt_t_7; + float dq_o_7 = dq2_7 + dqf_7; + float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_24[0]; + float qe_6 = q_t_1_7 * e_dg_7; + float kp_18 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_7) * _tmem_load_23[0]); + float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_24[0], k_t_7, cr_6)) + dgf_7; + s_run_1302[0] = s_run_1302[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_15 = t_15 * qrs; - if (t_15 < clen) { + int qoff_7 = t_7 * qrs; + if (t_7 < clen) { { - auto _gst_410 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_410)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_414 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_414)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_414) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_411 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_411)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_415 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_415)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_415) + (0)) = (__nv_bfloat16)dk_o_7; } } float tot_1471 = s_run_1302[0]; @@ -4859,40 +4484,40 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM int b3_1473 = lane_1 >> 3 & 1; int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; - float send_6 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_412 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_412; - float send_7 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_413; - float send_8 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_414 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_414; - float send_9 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_415; + float send_8 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); + float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_416; + float send_9 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_417; + float send_10 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); + float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_418; + float send_11 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, send_11, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_419; float v2_1480[2]; - float send_10 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_416; - float send_11 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_417; + float send_12 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); + float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_420; + float send_13 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, send_13, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_421; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_418; - float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_419; - float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_420; + float _shfl_xor_sync_422 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_422; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_423; + float _shfl_xor_sync_424 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_424; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_421 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_421), "f"(tot_1471) : "memory"); + uint32_t _addr_425 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_425), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4924,132 +4549,132 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_422 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_426 = make_float2(gacc[4], gacc[4]); float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_422), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_425; - asm("abs.f32 %0, %1;" : "=f"(_abs_425) : "f"(rp_z_7.x)); - float _neg_424; - asm("neg.f32 %0, %1;" : "=f"(_neg_424) : "f"(_abs_425)); - float _exp_423 = __nv_expf(_neg_424); - float _abs_428; - asm("abs.f32 %0, %1;" : "=f"(_abs_428) : "f"(rp_z_7.y)); - float _neg_427; - asm("neg.f32 %0, %1;" : "=f"(_neg_427) : "f"(_abs_428)); - float _exp_426 = __nv_expf(_neg_427); - float2 rp_e_7 = make_float2(_exp_423, _exp_426); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_426), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_429; + asm("abs.f32 %0, %1;" : "=f"(_abs_429) : "f"(rp_z_7.x)); + float _neg_428; + asm("neg.f32 %0, %1;" : "=f"(_neg_428) : "f"(_abs_429)); + float _exp_427 = __nv_expf(_neg_428); + float _abs_432; + asm("abs.f32 %0, %1;" : "=f"(_abs_432) : "f"(rp_z_7.y)); + float _neg_431; + asm("neg.f32 %0, %1;" : "=f"(_neg_431) : "f"(_abs_432)); + float _exp_430 = __nv_expf(_neg_431); + float2 rp_e_7 = make_float2(_exp_427, _exp_430); float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_429 = approx_rcp(rp_den_7.x); - float _rcp_430 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_431 = make_float2(_rcp_429, _rcp_430); + float _rcp_433 = approx_rcp(rp_den_7.x); + float _rcp_434 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_435 = make_float2(_rcp_433, _rcp_434); float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_431)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_435)); float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_432 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_436 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_432)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_436)); u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_433 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_437 = make_float2(gacc[4], gacc[4]); float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_433), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_436; - asm("abs.f32 %0, %1;" : "=f"(_abs_436) : "f"(rp_z_8.x)); - float _neg_435; - asm("neg.f32 %0, %1;" : "=f"(_neg_435) : "f"(_abs_436)); - float _exp_434 = __nv_expf(_neg_435); - float _abs_439; - asm("abs.f32 %0, %1;" : "=f"(_abs_439) : "f"(rp_z_8.y)); - float _neg_438; - asm("neg.f32 %0, %1;" : "=f"(_neg_438) : "f"(_abs_439)); - float _exp_437 = __nv_expf(_neg_438); - float2 rp_e_8 = make_float2(_exp_434, _exp_437); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_437), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_440; + asm("abs.f32 %0, %1;" : "=f"(_abs_440) : "f"(rp_z_8.x)); + float _neg_439; + asm("neg.f32 %0, %1;" : "=f"(_neg_439) : "f"(_abs_440)); + float _exp_438 = __nv_expf(_neg_439); + float _abs_443; + asm("abs.f32 %0, %1;" : "=f"(_abs_443) : "f"(rp_z_8.y)); + float _neg_442; + asm("neg.f32 %0, %1;" : "=f"(_neg_442) : "f"(_abs_443)); + float _exp_441 = __nv_expf(_neg_442); + float2 rp_e_8 = make_float2(_exp_438, _exp_441); float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_440 = approx_rcp(rp_den_8.x); - float _rcp_441 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_442 = make_float2(_rcp_440, _rcp_441); + float _rcp_444 = approx_rcp(rp_den_8.x); + float _rcp_445 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_446 = make_float2(_rcp_444, _rcp_445); float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_442)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_446)); float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_443 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_447 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_443)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_447)); u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_444 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_448 = make_float2(gacc[4], gacc[4]); float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_444), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_447; - asm("abs.f32 %0, %1;" : "=f"(_abs_447) : "f"(rp_z_9.x)); - float _neg_446; - asm("neg.f32 %0, %1;" : "=f"(_neg_446) : "f"(_abs_447)); - float _exp_445 = __nv_expf(_neg_446); - float _abs_450; - asm("abs.f32 %0, %1;" : "=f"(_abs_450) : "f"(rp_z_9.y)); - float _neg_449; - asm("neg.f32 %0, %1;" : "=f"(_neg_449) : "f"(_abs_450)); - float _exp_448 = __nv_expf(_neg_449); - float2 rp_e_9 = make_float2(_exp_445, _exp_448); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_448), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_451; + asm("abs.f32 %0, %1;" : "=f"(_abs_451) : "f"(rp_z_9.x)); + float _neg_450; + asm("neg.f32 %0, %1;" : "=f"(_neg_450) : "f"(_abs_451)); + float _exp_449 = __nv_expf(_neg_450); + float _abs_454; + asm("abs.f32 %0, %1;" : "=f"(_abs_454) : "f"(rp_z_9.y)); + float _neg_453; + asm("neg.f32 %0, %1;" : "=f"(_neg_453) : "f"(_abs_454)); + float _exp_452 = __nv_expf(_neg_453); + float2 rp_e_9 = make_float2(_exp_449, _exp_452); float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_451 = approx_rcp(rp_den_9.x); - float _rcp_452 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_453 = make_float2(_rcp_451, _rcp_452); + float _rcp_455 = approx_rcp(rp_den_9.x); + float _rcp_456 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_457 = make_float2(_rcp_455, _rcp_456); float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_453)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_457)); float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_454 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_458 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_454)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_455 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_459 = make_float2(gacc[4], gacc[4]); float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_455), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_458; - asm("abs.f32 %0, %1;" : "=f"(_abs_458) : "f"(rp_z_10.x)); - float _neg_457; - asm("neg.f32 %0, %1;" : "=f"(_neg_457) : "f"(_abs_458)); - float _exp_456 = __nv_expf(_neg_457); - float _abs_461; - asm("abs.f32 %0, %1;" : "=f"(_abs_461) : "f"(rp_z_10.y)); - float _neg_460; - asm("neg.f32 %0, %1;" : "=f"(_neg_460) : "f"(_abs_461)); - float _exp_459 = __nv_expf(_neg_460); - float2 rp_e_10 = make_float2(_exp_456, _exp_459); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_459), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_462; + asm("abs.f32 %0, %1;" : "=f"(_abs_462) : "f"(rp_z_10.x)); + float _neg_461; + asm("neg.f32 %0, %1;" : "=f"(_neg_461) : "f"(_abs_462)); + float _exp_460 = __nv_expf(_neg_461); + float _abs_465; + asm("abs.f32 %0, %1;" : "=f"(_abs_465) : "f"(rp_z_10.y)); + float _neg_464; + asm("neg.f32 %0, %1;" : "=f"(_neg_464) : "f"(_abs_465)); + float _exp_463 = __nv_expf(_neg_464); + float2 rp_e_10 = make_float2(_exp_460, _exp_463); float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_462 = approx_rcp(rp_den_10.x); - float _rcp_463 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_464 = make_float2(_rcp_462, _rcp_463); + float _rcp_466 = approx_rcp(rp_den_10.x); + float _rcp_467 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_468 = make_float2(_rcp_466, _rcp_467); float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_464)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_468)); float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_465 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_469 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_465)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -5064,54 +4689,54 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_466; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_466) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_466; + uint32_t _prmt_470; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_470; int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_467 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_467)); - *(reinterpret_cast(_gst_467) + (0)) = pq_8[0]; + auto _gst_471 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_471)); + *(reinterpret_cast(_gst_471) + (0)) = pq_8[0]; } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_468; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_468) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_468; + uint32_t _prmt_472; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_472; int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_469 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_469)); - *(reinterpret_cast(_gst_469) + (0)) = pq_9[0]; + auto _gst_473 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_473)); + *(reinterpret_cast(_gst_473) + (0)) = pq_9[0]; } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_470; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_470; + uint32_t _prmt_474; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_474) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_474; int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_471 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_471)); - *(reinterpret_cast(_gst_471) + (0)) = pq_10[0]; + auto _gst_475 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_475)); + *(reinterpret_cast(_gst_475) + (0)) = pq_10[0]; } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_472; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_472; + uint32_t _prmt_476; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_476) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_476; int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_473 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_473)); - *(reinterpret_cast(_gst_473) + (0)) = pq_11[0]; + auto _gst_477 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_477)); + *(reinterpret_cast(_gst_477) + (0)) = pq_11[0]; } } int t_a_16 = 16 + 8 * th; @@ -5176,47 +4801,47 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_474; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_474) : "f"(s_a_1), "f"(nm_t_3)); + float _sub_478; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_478) : "f"(s_a_1), "f"(nm_t_3)); float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_474), "f"(pa_1)); - float _sub_475; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_475) : "f"(pa_1), "f"(nm_t_3)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_478), "f"(pa_1)); + float _sub_479; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_479) : "f"(pa_1), "f"(nm_t_3)); float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_475), "f"(s_a_1)); - float _abs_477; - asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(s_a_1)); - float _abs_478; - asm("abs.f32 %0, %1;" : "=f"(_abs_478) : "f"(pa_1)); - uint32_t _setp_476_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_476_bits) : "f"(_abs_477), "f"(_abs_478)); - bool _setp_476 = _setp_476_bits != 0; - float nm_err_3 = ((_setp_476) ? nm_es_3 : nm_ex_3); - float _add_479; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_479) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_479; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_479), "f"(s_a_1)); + float _abs_481; + asm("abs.f32 %0, %1;" : "=f"(_abs_481) : "f"(s_a_1)); + float _abs_482; + asm("abs.f32 %0, %1;" : "=f"(_abs_482) : "f"(pa_1)); + uint32_t _setp_480_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_480_bits) : "f"(_abs_481), "f"(_abs_482)); + bool _setp_480 = _setp_480_bits != 0; + float nm_err_3 = ((_setp_480) ? nm_es_3 : nm_ex_3); + float _add_483; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_483) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_483; s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_480; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_480) : "f"(s_b_1), "f"(nm_t_4)); + float _sub_484; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_484) : "f"(s_b_1), "f"(nm_t_4)); float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_480), "f"(pb_1)); - float _sub_481; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_481) : "f"(pb_1), "f"(nm_t_4)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_484), "f"(pb_1)); + float _sub_485; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_485) : "f"(pb_1), "f"(nm_t_4)); float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_481), "f"(s_b_1)); - float _abs_483; - asm("abs.f32 %0, %1;" : "=f"(_abs_483) : "f"(s_b_1)); - float _abs_484; - asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(pb_1)); - uint32_t _setp_482_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_482_bits) : "f"(_abs_483), "f"(_abs_484)); - bool _setp_482 = _setp_482_bits != 0; - float nm_err_4 = ((_setp_482) ? nm_es_4 : nm_ex_4); - float _add_485; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_485) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_485; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_485), "f"(s_b_1)); + float _abs_487; + asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(s_b_1)); + float _abs_488; + asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(pb_1)); + uint32_t _setp_486_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_486_bits) : "f"(_abs_487), "f"(_abs_488)); + bool _setp_486 = _setp_486_bits != 0; + float nm_err_4 = ((_setp_486) ? nm_es_4 : nm_ex_4); + float _add_489; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_489) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_489; s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5224,8 +4849,8 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_486 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_486), "f"(off_acc[0]) : "memory"); + uint32_t _addr_490 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_490), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5249,9 +4874,9 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1514 = r_322[0]; + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1514 = r_280[0]; float gc_1515 = gn_1514 - 21.640425f; int eb_1516 = 98304 + th * -32768; int ew_1517 = eb_1516 >> 2; @@ -5276,348 +4901,388 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1524[1]; s_run_1524[0] = 0.0f; - int t_16 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1517 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1517 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1517 + 2944 + d_lane]; - float bt_t_16 = btv_1523[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1514); - float e_dg_16 = approx_exp2(g_t_16 - gc_1515); - float dq2_16 = _tmem_load_26[7] * e_dg_16; - float dk2_16 = _tmem_load_27[7] * e_dg_16; - dbp_1522[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_28[7]; - float dg_o_16 = __fmaf_rn(q_t_1_24, dq2_16, (dk2b_16 - _tmem_load_28[7]) * k_t_16) + dgf_16; - s_run_1524[0] = s_run_1524[0] + dg_o_16; + int t_8 = 8 * th + 7; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_8 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_8 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_8 = __uint_as_float(r_283[0] << 16); + float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); + float dqf_8 = pool_f32[ew_1517 + 896 + d_lane]; + float dkf_8 = pool_f32[ew_1517 + 1920 + d_lane]; + float dgf_8 = pool_f32[ew_1517 + 2944 + d_lane]; + float bt_t_8 = btv_1523[7]; + float e_off_8 = approx_exp2(g_t_8 - gn_1514); + float e_dg_8 = approx_exp2(g_t_8 - gc_1515); + float dq2_8 = _tmem_load_26[7] * e_dg_8; + float dk2_8 = _tmem_load_27[7] * e_dg_8; + dbp_1522[7] = dk2_8 * k_t_8; + float dk2b_8 = dk2_8 * bt_t_8; + float dq_o_8 = dq2_8 + dqf_8; + float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_28[7]; + float qe_7 = q_t_1_8 * e_dg_8; + float kp_19 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_8) * _tmem_load_27[7]); + float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_28[7], k_t_8, cr_7)) + dgf_8; + s_run_1524[0] = s_run_1524[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_16 = t_16 * qrs; - if (t_16 < clen) { + int qoff_8 = t_8 * qrs; + if (t_8 < clen) { { - auto _gst_487 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_487)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_487) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_491 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_491)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_488 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_488)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_488) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_492 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_492)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_8; } } - int t_17 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1517 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1517 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1517 + 2816 + d_lane]; - float bt_t_17 = btv_1523[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1514); - float e_dg_17 = approx_exp2(g_t_17 - gc_1515); - float dq2_17 = _tmem_load_26[6] * e_dg_17; - float dk2_17 = _tmem_load_27[6] * e_dg_17; - dbp_1522[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[6]; - float dg_o_17 = __fmaf_rn(q_t_1_25, dq2_17, (dk2b_17 - _tmem_load_28[6]) * k_t_17) + dgf_17; - s_run_1524[0] = s_run_1524[0] + dg_o_17; + int t_9 = 8 * th + 6; + float r_284[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_9 = r_284[0]; + unsigned int r_285[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_9 = __uint_as_float(r_285[0] << 16); + unsigned int r_286[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_9 = __uint_as_float(r_286[0] << 16); + float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); + float dqf_9 = pool_f32[ew_1517 + 768 + d_lane]; + float dkf_9 = pool_f32[ew_1517 + 1792 + d_lane]; + float dgf_9 = pool_f32[ew_1517 + 2816 + d_lane]; + float bt_t_9 = btv_1523[6]; + float e_off_9 = approx_exp2(g_t_9 - gn_1514); + float e_dg_9 = approx_exp2(g_t_9 - gc_1515); + float dq2_9 = _tmem_load_26[6] * e_dg_9; + float dk2_9 = _tmem_load_27[6] * e_dg_9; + dbp_1522[6] = dk2_9 * k_t_9; + float dk2b_9 = dk2_9 * bt_t_9; + float dq_o_9 = dq2_9 + dqf_9; + float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_28[6]; + float qe_8 = q_t_1_9 * e_dg_9; + float kp_20 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_9) * _tmem_load_27[6]); + float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_28[6], k_t_9, cr_8)) + dgf_9; + s_run_1524[0] = s_run_1524[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_17 = t_17 * qrs; - if (t_17 < clen) { + int qoff_9 = t_9 * qrs; + if (t_9 < clen) { { - auto _gst_489 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_489)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_489) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_493 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_493)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_490 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_490)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_490) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_494 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_494)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_9; } } - int t_18 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1517 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1517 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1517 + 2688 + d_lane]; - float bt_t_18 = btv_1523[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1514); - float e_dg_18 = approx_exp2(g_t_18 - gc_1515); - float dq2_18 = _tmem_load_26[5] * e_dg_18; - float dk2_18 = _tmem_load_27[5] * e_dg_18; - dbp_1522[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[5]; - float dg_o_18 = __fmaf_rn(q_t_1_26, dq2_18, (dk2b_18 - _tmem_load_28[5]) * k_t_18) + dgf_18; - s_run_1524[0] = s_run_1524[0] + dg_o_18; + int t_10 = 8 * th + 5; + float r_287[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_10 = r_287[0]; + unsigned int r_288[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_10 = __uint_as_float(r_288[0] << 16); + unsigned int r_289[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_10 = __uint_as_float(r_289[0] << 16); + float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); + float dqf_10 = pool_f32[ew_1517 + 640 + d_lane]; + float dkf_10 = pool_f32[ew_1517 + 1664 + d_lane]; + float dgf_10 = pool_f32[ew_1517 + 2688 + d_lane]; + float bt_t_10 = btv_1523[5]; + float e_off_10 = approx_exp2(g_t_10 - gn_1514); + float e_dg_10 = approx_exp2(g_t_10 - gc_1515); + float dq2_10 = _tmem_load_26[5] * e_dg_10; + float dk2_10 = _tmem_load_27[5] * e_dg_10; + dbp_1522[5] = dk2_10 * k_t_10; + float dk2b_10 = dk2_10 * bt_t_10; + float dq_o_10 = dq2_10 + dqf_10; + float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_28[5]; + float qe_9 = q_t_1_10 * e_dg_10; + float kp_21 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_10) * _tmem_load_27[5]); + float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_28[5], k_t_10, cr_9)) + dgf_10; + s_run_1524[0] = s_run_1524[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_18 = t_18 * qrs; - if (t_18 < clen) { + int qoff_10 = t_10 * qrs; + if (t_10 < clen) { { - auto _gst_491 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_491)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_495 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_495)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_492 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_492)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_496 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_496)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_10; } } - int t_19 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1517 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1517 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1517 + 2560 + d_lane]; - float bt_t_19 = btv_1523[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1514); - float e_dg_19 = approx_exp2(g_t_19 - gc_1515); - float dq2_19 = _tmem_load_26[4] * e_dg_19; - float dk2_19 = _tmem_load_27[4] * e_dg_19; - dbp_1522[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[4]; - float dg_o_19 = __fmaf_rn(q_t_1_27, dq2_19, (dk2b_19 - _tmem_load_28[4]) * k_t_19) + dgf_19; - s_run_1524[0] = s_run_1524[0] + dg_o_19; + int t_11 = 8 * th + 4; + float r_290[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_11 = r_290[0]; + unsigned int r_291[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_11 = __uint_as_float(r_291[0] << 16); + unsigned int r_292[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_11 = __uint_as_float(r_292[0] << 16); + float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); + float dqf_11 = pool_f32[ew_1517 + 512 + d_lane]; + float dkf_11 = pool_f32[ew_1517 + 1536 + d_lane]; + float dgf_11 = pool_f32[ew_1517 + 2560 + d_lane]; + float bt_t_11 = btv_1523[4]; + float e_off_11 = approx_exp2(g_t_11 - gn_1514); + float e_dg_11 = approx_exp2(g_t_11 - gc_1515); + float dq2_11 = _tmem_load_26[4] * e_dg_11; + float dk2_11 = _tmem_load_27[4] * e_dg_11; + dbp_1522[4] = dk2_11 * k_t_11; + float dk2b_11 = dk2_11 * bt_t_11; + float dq_o_11 = dq2_11 + dqf_11; + float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_28[4]; + float qe_10 = q_t_1_11 * e_dg_11; + float kp_22 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_11) * _tmem_load_27[4]); + float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_28[4], k_t_11, cr_10)) + dgf_11; + s_run_1524[0] = s_run_1524[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_19 = t_19 * qrs; - if (t_19 < clen) { + int qoff_11 = t_11 * qrs; + if (t_11 < clen) { { - auto _gst_493 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_493)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_497 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_497)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_494 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_494)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_498 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_498)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_11; } } - int t_20 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1517 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1517 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1517 + 2432 + d_lane]; - float bt_t_20 = btv_1523[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1514); - float e_dg_20 = approx_exp2(g_t_20 - gc_1515); - float dq2_20 = _tmem_load_26[3] * e_dg_20; - float dk2_20 = _tmem_load_27[3] * e_dg_20; - dbp_1522[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[3]; - float dg_o_20 = __fmaf_rn(q_t_1_28, dq2_20, (dk2b_20 - _tmem_load_28[3]) * k_t_20) + dgf_20; - s_run_1524[0] = s_run_1524[0] + dg_o_20; + int t_12 = 8 * th + 3; + float r_293[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_12 = r_293[0]; + unsigned int r_294[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_12 = __uint_as_float(r_294[0] << 16); + unsigned int r_295[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_12 = __uint_as_float(r_295[0] << 16); + float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); + float dqf_12 = pool_f32[ew_1517 + 384 + d_lane]; + float dkf_12 = pool_f32[ew_1517 + 1408 + d_lane]; + float dgf_12 = pool_f32[ew_1517 + 2432 + d_lane]; + float bt_t_12 = btv_1523[3]; + float e_off_12 = approx_exp2(g_t_12 - gn_1514); + float e_dg_12 = approx_exp2(g_t_12 - gc_1515); + float dq2_12 = _tmem_load_26[3] * e_dg_12; + float dk2_12 = _tmem_load_27[3] * e_dg_12; + dbp_1522[3] = dk2_12 * k_t_12; + float dk2b_12 = dk2_12 * bt_t_12; + float dq_o_12 = dq2_12 + dqf_12; + float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_28[3]; + float qe_11 = q_t_1_12 * e_dg_12; + float kp_23 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_12) * _tmem_load_27[3]); + float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_28[3], k_t_12, cr_11)) + dgf_12; + s_run_1524[0] = s_run_1524[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_20 = t_20 * qrs; - if (t_20 < clen) { + int qoff_12 = t_12 * qrs; + if (t_12 < clen) { { - auto _gst_495 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_495)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_499 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_499)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_496 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_496)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_500 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_500)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_12; } } - int t_21 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1517 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1517 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1517 + 2304 + d_lane]; - float bt_t_21 = btv_1523[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1514); - float e_dg_21 = approx_exp2(g_t_21 - gc_1515); - float dq2_21 = _tmem_load_26[2] * e_dg_21; - float dk2_21 = _tmem_load_27[2] * e_dg_21; - dbp_1522[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[2]; - float dg_o_21 = __fmaf_rn(q_t_1_29, dq2_21, (dk2b_21 - _tmem_load_28[2]) * k_t_21) + dgf_21; - s_run_1524[0] = s_run_1524[0] + dg_o_21; + int t_13 = 8 * th + 2; + float r_296[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_13 = r_296[0]; + unsigned int r_297[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_13 = __uint_as_float(r_297[0] << 16); + unsigned int r_298[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_13 = __uint_as_float(r_298[0] << 16); + float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); + float dqf_13 = pool_f32[ew_1517 + 256 + d_lane]; + float dkf_13 = pool_f32[ew_1517 + 1280 + d_lane]; + float dgf_13 = pool_f32[ew_1517 + 2304 + d_lane]; + float bt_t_13 = btv_1523[2]; + float e_off_13 = approx_exp2(g_t_13 - gn_1514); + float e_dg_13 = approx_exp2(g_t_13 - gc_1515); + float dq2_13 = _tmem_load_26[2] * e_dg_13; + float dk2_13 = _tmem_load_27[2] * e_dg_13; + dbp_1522[2] = dk2_13 * k_t_13; + float dk2b_13 = dk2_13 * bt_t_13; + float dq_o_13 = dq2_13 + dqf_13; + float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_28[2]; + float qe_12 = q_t_1_13 * e_dg_13; + float kp_24 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_13) * _tmem_load_27[2]); + float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_28[2], k_t_13, cr_12)) + dgf_13; + s_run_1524[0] = s_run_1524[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_21 = t_21 * qrs; - if (t_21 < clen) { + int qoff_13 = t_13 * qrs; + if (t_13 < clen) { { - auto _gst_497 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_497)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_501 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_501)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_498 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_498)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_502 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_502)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_13; } } - int t_22 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1517 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1517 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1517 + 2176 + d_lane]; - float bt_t_22 = btv_1523[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1514); - float e_dg_22 = approx_exp2(g_t_22 - gc_1515); - float dq2_22 = _tmem_load_26[1] * e_dg_22; - float dk2_22 = _tmem_load_27[1] * e_dg_22; - dbp_1522[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[1]; - float dg_o_22 = __fmaf_rn(q_t_1_30, dq2_22, (dk2b_22 - _tmem_load_28[1]) * k_t_22) + dgf_22; - s_run_1524[0] = s_run_1524[0] + dg_o_22; + int t_14 = 8 * th + 1; + float r_299[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_14 = r_299[0]; + unsigned int r_300[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_14 = __uint_as_float(r_300[0] << 16); + unsigned int r_301[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_14 = __uint_as_float(r_301[0] << 16); + float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); + float dqf_14 = pool_f32[ew_1517 + 128 + d_lane]; + float dkf_14 = pool_f32[ew_1517 + 1152 + d_lane]; + float dgf_14 = pool_f32[ew_1517 + 2176 + d_lane]; + float bt_t_14 = btv_1523[1]; + float e_off_14 = approx_exp2(g_t_14 - gn_1514); + float e_dg_14 = approx_exp2(g_t_14 - gc_1515); + float dq2_14 = _tmem_load_26[1] * e_dg_14; + float dk2_14 = _tmem_load_27[1] * e_dg_14; + dbp_1522[1] = dk2_14 * k_t_14; + float dk2b_14 = dk2_14 * bt_t_14; + float dq_o_14 = dq2_14 + dqf_14; + float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_28[1]; + float qe_13 = q_t_1_14 * e_dg_14; + float kp_25 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_14) * _tmem_load_27[1]); + float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_28[1], k_t_14, cr_13)) + dgf_14; + s_run_1524[0] = s_run_1524[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_22 = t_22 * qrs; - if (t_22 < clen) { + int qoff_14 = t_14 * qrs; + if (t_14 < clen) { { - auto _gst_499 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_499)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_503 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_503)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_503) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_500 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_500)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_504 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_504)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_504) + (0)) = (__nv_bfloat16)dk_o_14; } } - int t_23 = 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1517 + d_lane]; - float dkf_23 = pool_f32[ew_1517 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1517 + 2048 + d_lane]; - float bt_t_23 = btv_1523[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1514); - float e_dg_23 = approx_exp2(g_t_23 - gc_1515); - float dq2_23 = _tmem_load_26[0] * e_dg_23; - float dk2_23 = _tmem_load_27[0] * e_dg_23; - dbp_1522[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[0]; - float dg_o_23 = __fmaf_rn(q_t_1_31, dq2_23, (dk2b_23 - _tmem_load_28[0]) * k_t_23) + dgf_23; - s_run_1524[0] = s_run_1524[0] + dg_o_23; + int t_15 = 8 * th; + float r_302[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_15 = r_302[0]; + unsigned int r_303[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_15 = __uint_as_float(r_303[0] << 16); + unsigned int r_304[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_15 = __uint_as_float(r_304[0] << 16); + float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); + float dqf_15 = pool_f32[ew_1517 + d_lane]; + float dkf_15 = pool_f32[ew_1517 + 1024 + d_lane]; + float dgf_15 = pool_f32[ew_1517 + 2048 + d_lane]; + float bt_t_15 = btv_1523[0]; + float e_off_15 = approx_exp2(g_t_15 - gn_1514); + float e_dg_15 = approx_exp2(g_t_15 - gc_1515); + float dq2_15 = _tmem_load_26[0] * e_dg_15; + float dk2_15 = _tmem_load_27[0] * e_dg_15; + dbp_1522[0] = dk2_15 * k_t_15; + float dk2b_15 = dk2_15 * bt_t_15; + float dq_o_15 = dq2_15 + dqf_15; + float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_28[0]; + float qe_14 = q_t_1_15 * e_dg_15; + float kp_26 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_15) * _tmem_load_27[0]); + float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_28[0], k_t_15, cr_14)) + dgf_15; + s_run_1524[0] = s_run_1524[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_23 = t_23 * qrs; - if (t_23 < clen) { + int qoff_15 = t_15 * qrs; + if (t_15 < clen) { { - auto _gst_501 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_501)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_505 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_505)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_505) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_502 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_502)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_506 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_506)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_506) + (0)) = (__nv_bfloat16)dk_o_15; } } float tot_1693 = s_run_1524[0]; @@ -5625,38 +5290,38 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM int b3_1695 = lane_1 >> 3 & 1; int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; - float send_12 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_503 = __shfl_xor_sync(0xFFFFFFFF, send_12, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_503; - float send_13 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_504; - float send_14 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_505 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_505; - float send_15 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_506; + float send_14 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); + float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_507; + float send_15 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); + float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_508; + float send_16 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); + float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_509; + float send_17 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); + float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, send_17, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_510; float v2_1702[2]; - float send_16 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_16, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_507; - float send_17 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_508; + float send_18 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); + float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_511; + float send_19 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); + float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, send_19, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_512; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_509; - float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_510; - float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_511; + float _shfl_xor_sync_513 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_513; + float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_514; + float _shfl_xor_sync_515 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_515; tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_512 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_512), "f"(tot_1693) : "memory"); + uint32_t _addr_516 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_516), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5688,132 +5353,132 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_513 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_517 = make_float2(gacc[4], gacc[4]); float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_513), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_516; - asm("abs.f32 %0, %1;" : "=f"(_abs_516) : "f"(rp_z_11.x)); - float _neg_515; - asm("neg.f32 %0, %1;" : "=f"(_neg_515) : "f"(_abs_516)); - float _exp_514 = __nv_expf(_neg_515); - float _abs_519; - asm("abs.f32 %0, %1;" : "=f"(_abs_519) : "f"(rp_z_11.y)); - float _neg_518; - asm("neg.f32 %0, %1;" : "=f"(_neg_518) : "f"(_abs_519)); - float _exp_517 = __nv_expf(_neg_518); - float2 rp_e_11 = make_float2(_exp_514, _exp_517); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_517), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_520; + asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_11.x)); + float _neg_519; + asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); + float _exp_518 = __nv_expf(_neg_519); + float _abs_523; + asm("abs.f32 %0, %1;" : "=f"(_abs_523) : "f"(rp_z_11.y)); + float _neg_522; + asm("neg.f32 %0, %1;" : "=f"(_neg_522) : "f"(_abs_523)); + float _exp_521 = __nv_expf(_neg_522); + float2 rp_e_11 = make_float2(_exp_518, _exp_521); float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_520 = approx_rcp(rp_den_11.x); - float _rcp_521 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_522 = make_float2(_rcp_520, _rcp_521); + float _rcp_524 = approx_rcp(rp_den_11.x); + float _rcp_525 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_526 = make_float2(_rcp_524, _rcp_525); float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_522)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_526)); float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_523 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_527 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_527)); u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_524 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_528 = make_float2(gacc[4], gacc[4]); float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_524), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_527; - asm("abs.f32 %0, %1;" : "=f"(_abs_527) : "f"(rp_z_12.x)); - float _neg_526; - asm("neg.f32 %0, %1;" : "=f"(_neg_526) : "f"(_abs_527)); - float _exp_525 = __nv_expf(_neg_526); - float _abs_530; - asm("abs.f32 %0, %1;" : "=f"(_abs_530) : "f"(rp_z_12.y)); - float _neg_529; - asm("neg.f32 %0, %1;" : "=f"(_neg_529) : "f"(_abs_530)); - float _exp_528 = __nv_expf(_neg_529); - float2 rp_e_12 = make_float2(_exp_525, _exp_528); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_528), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_531; + asm("abs.f32 %0, %1;" : "=f"(_abs_531) : "f"(rp_z_12.x)); + float _neg_530; + asm("neg.f32 %0, %1;" : "=f"(_neg_530) : "f"(_abs_531)); + float _exp_529 = __nv_expf(_neg_530); + float _abs_534; + asm("abs.f32 %0, %1;" : "=f"(_abs_534) : "f"(rp_z_12.y)); + float _neg_533; + asm("neg.f32 %0, %1;" : "=f"(_neg_533) : "f"(_abs_534)); + float _exp_532 = __nv_expf(_neg_533); + float2 rp_e_12 = make_float2(_exp_529, _exp_532); float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_531 = approx_rcp(rp_den_12.x); - float _rcp_532 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_533 = make_float2(_rcp_531, _rcp_532); + float _rcp_535 = approx_rcp(rp_den_12.x); + float _rcp_536 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_537 = make_float2(_rcp_535, _rcp_536); float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_533)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_537)); float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_534 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_538 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_534)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_538)); u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_535 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_539 = make_float2(gacc[4], gacc[4]); float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_535), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_538; - asm("abs.f32 %0, %1;" : "=f"(_abs_538) : "f"(rp_z_13.x)); - float _neg_537; - asm("neg.f32 %0, %1;" : "=f"(_neg_537) : "f"(_abs_538)); - float _exp_536 = __nv_expf(_neg_537); - float _abs_541; - asm("abs.f32 %0, %1;" : "=f"(_abs_541) : "f"(rp_z_13.y)); - float _neg_540; - asm("neg.f32 %0, %1;" : "=f"(_neg_540) : "f"(_abs_541)); - float _exp_539 = __nv_expf(_neg_540); - float2 rp_e_13 = make_float2(_exp_536, _exp_539); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_539), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_542; + asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(rp_z_13.x)); + float _neg_541; + asm("neg.f32 %0, %1;" : "=f"(_neg_541) : "f"(_abs_542)); + float _exp_540 = __nv_expf(_neg_541); + float _abs_545; + asm("abs.f32 %0, %1;" : "=f"(_abs_545) : "f"(rp_z_13.y)); + float _neg_544; + asm("neg.f32 %0, %1;" : "=f"(_neg_544) : "f"(_abs_545)); + float _exp_543 = __nv_expf(_neg_544); + float2 rp_e_13 = make_float2(_exp_540, _exp_543); float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_542 = approx_rcp(rp_den_13.x); - float _rcp_543 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_544 = make_float2(_rcp_542, _rcp_543); + float _rcp_546 = approx_rcp(rp_den_13.x); + float _rcp_547 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_548 = make_float2(_rcp_546, _rcp_547); float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_544)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_548)); float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_545 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_549 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_545)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_549)); u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_546 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_550 = make_float2(gacc[4], gacc[4]); float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_546), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_549; - asm("abs.f32 %0, %1;" : "=f"(_abs_549) : "f"(rp_z_14.x)); - float _neg_548; - asm("neg.f32 %0, %1;" : "=f"(_neg_548) : "f"(_abs_549)); - float _exp_547 = __nv_expf(_neg_548); - float _abs_552; - asm("abs.f32 %0, %1;" : "=f"(_abs_552) : "f"(rp_z_14.y)); - float _neg_551; - asm("neg.f32 %0, %1;" : "=f"(_neg_551) : "f"(_abs_552)); - float _exp_550 = __nv_expf(_neg_551); - float2 rp_e_14 = make_float2(_exp_547, _exp_550); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_550), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_553; + asm("abs.f32 %0, %1;" : "=f"(_abs_553) : "f"(rp_z_14.x)); + float _neg_552; + asm("neg.f32 %0, %1;" : "=f"(_neg_552) : "f"(_abs_553)); + float _exp_551 = __nv_expf(_neg_552); + float _abs_556; + asm("abs.f32 %0, %1;" : "=f"(_abs_556) : "f"(rp_z_14.y)); + float _neg_555; + asm("neg.f32 %0, %1;" : "=f"(_neg_555) : "f"(_abs_556)); + float _exp_554 = __nv_expf(_neg_555); + float2 rp_e_14 = make_float2(_exp_551, _exp_554); float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_553 = approx_rcp(rp_den_14.x); - float _rcp_554 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_555 = make_float2(_rcp_553, _rcp_554); + float _rcp_557 = approx_rcp(rp_den_14.x); + float _rcp_558 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_559 = make_float2(_rcp_557, _rcp_558); float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_555)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_559)); float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_556 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_560 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_556)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_560)); u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -5828,54 +5493,54 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_557; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_557) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_557; + uint32_t _prmt_561; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_561; int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_558 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_558)); - *(reinterpret_cast(_gst_558) + (0)) = pq_12[0]; + auto _gst_562 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_562)); + *(reinterpret_cast(_gst_562) + (0)) = pq_12[0]; } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_559; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_559) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_559; + uint32_t _prmt_563; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_563; int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_560 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_560)); - *(reinterpret_cast(_gst_560) + (0)) = pq_13[0]; + auto _gst_564 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_564)); + *(reinterpret_cast(_gst_564) + (0)) = pq_13[0]; } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_561; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_561; + uint32_t _prmt_565; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_565) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_565; int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_562 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_562)); - *(reinterpret_cast(_gst_562) + (0)) = pq_14[0]; + auto _gst_566 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_566)); + *(reinterpret_cast(_gst_566) + (0)) = pq_14[0]; } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_563; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_563; + uint32_t _prmt_567; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_567) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_567; int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_564 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_564)); - *(reinterpret_cast(_gst_564) + (0)) = pq_15[0]; + auto _gst_568 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_568)); + *(reinterpret_cast(_gst_568) + (0)) = pq_15[0]; } } int t_a_24 = 8 * th; @@ -5940,47 +5605,47 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_565; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_565) : "f"(s_a_2), "f"(nm_t_5)); + float _sub_569; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_569) : "f"(s_a_2), "f"(nm_t_5)); float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_565), "f"(pa_2)); - float _sub_566; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_566) : "f"(pa_2), "f"(nm_t_5)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_569), "f"(pa_2)); + float _sub_570; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_570) : "f"(pa_2), "f"(nm_t_5)); float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_566), "f"(s_a_2)); - float _abs_568; - asm("abs.f32 %0, %1;" : "=f"(_abs_568) : "f"(s_a_2)); - float _abs_569; - asm("abs.f32 %0, %1;" : "=f"(_abs_569) : "f"(pa_2)); - uint32_t _setp_567_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_567_bits) : "f"(_abs_568), "f"(_abs_569)); - bool _setp_567 = _setp_567_bits != 0; - float nm_err_5 = ((_setp_567) ? nm_es_5 : nm_ex_5); - float _add_570; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_570; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_570), "f"(s_a_2)); + float _abs_572; + asm("abs.f32 %0, %1;" : "=f"(_abs_572) : "f"(s_a_2)); + float _abs_573; + asm("abs.f32 %0, %1;" : "=f"(_abs_573) : "f"(pa_2)); + uint32_t _setp_571_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_571_bits) : "f"(_abs_572), "f"(_abs_573)); + bool _setp_571 = _setp_571_bits != 0; + float nm_err_5 = ((_setp_571) ? nm_es_5 : nm_ex_5); + float _add_574; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_574; s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_571; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(s_b_2), "f"(nm_t_6)); + float _sub_575; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_575) : "f"(s_b_2), "f"(nm_t_6)); float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_571), "f"(pb_2)); - float _sub_572; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_572) : "f"(pb_2), "f"(nm_t_6)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_575), "f"(pb_2)); + float _sub_576; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_576) : "f"(pb_2), "f"(nm_t_6)); float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_572), "f"(s_b_2)); - float _abs_574; - asm("abs.f32 %0, %1;" : "=f"(_abs_574) : "f"(s_b_2)); - float _abs_575; - asm("abs.f32 %0, %1;" : "=f"(_abs_575) : "f"(pb_2)); - uint32_t _setp_573_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_573_bits) : "f"(_abs_574), "f"(_abs_575)); - bool _setp_573 = _setp_573_bits != 0; - float nm_err_6 = ((_setp_573) ? nm_es_6 : nm_ex_6); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_576; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_576), "f"(s_b_2)); + float _abs_578; + asm("abs.f32 %0, %1;" : "=f"(_abs_578) : "f"(s_b_2)); + float _abs_579; + asm("abs.f32 %0, %1;" : "=f"(_abs_579) : "f"(pb_2)); + uint32_t _setp_577_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_577_bits) : "f"(_abs_578), "f"(_abs_579)); + bool _setp_577 = _setp_577_bits != 0; + float nm_err_6 = ((_setp_577) ? nm_es_6 : nm_ex_6); + float _add_580; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_580; s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -6019,60 +5684,60 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_578; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_578) : "f"(ts_s), "f"(ts_bb)); - float _sub_577; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_577) : "f"(s_a_3), "f"(_sub_578)); - float _sub_579; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_579) : "f"(other[0]), "f"(ts_bb)); + float _sub_582; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s), "f"(ts_bb)); + float _sub_581; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(s_a_3), "f"(_sub_582)); + float _sub_583; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(other[0]), "f"(ts_bb)); float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_577), "f"(_sub_579)); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_a_3), "f"(other[1])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_581), "f"(_sub_583)); + float _add_584; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(c_a_3), "f"(other[1])); float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_580), "f"(ts_e)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_584), "f"(ts_e)); float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_582; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_581; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(ts_s), "f"(_sub_582)); - float _sub_583; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(pm_c), "f"(ts_bb_0)); + float _sub_586; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_585; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s), "f"(_sub_586)); + float _sub_587; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_587) : "f"(pm_c), "f"(ts_bb_0)); float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_581), "f"(_sub_583)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_585), "f"(_sub_587)); s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_585; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_584; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_584) : "f"(s_b_3), "f"(_sub_585)); - float _sub_586; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(other[2]), "f"(ts_bb_1)); + float _sub_589; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_588; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(s_b_3), "f"(_sub_589)); + float _sub_590; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(other[2]), "f"(ts_bb_1)); float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_584), "f"(_sub_586)); - float _add_587; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_587) : "f"(c_b_3), "f"(other[3])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_588), "f"(_sub_590)); + float _add_591; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_591) : "f"(c_b_3), "f"(other[3])); float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_587), "f"(ts_e_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_591), "f"(ts_e_1)); float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_589; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_588; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(ts_s_1), "f"(_sub_589)); - float _sub_590; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(pm_c_0), "f"(ts_bb_2)); + float _sub_593; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_593) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_592; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_592) : "f"(ts_s_1), "f"(_sub_593)); + float _sub_594; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_594) : "f"(pm_c_0), "f"(ts_bb_2)); float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_588), "f"(_sub_590)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_592), "f"(_sub_594)); s_b_3 = ts_s_2; c_b_3 = ts_e_2; gacc[0] = s_a_3; @@ -6093,7 +5758,7 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_1706; pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1484; pool_f32[28160 + qd * 64 + 32 + 8 * th + e_star] = v1; - pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_1241; + pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_0; } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync 1, 256;" ::: "memory"); @@ -6183,41 +5848,41 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + for (int r_305 = 0; r_305 < 16; r_305++) { + int t_16 = r_305; + if (t_16 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6289,41 +5954,41 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_24 = 16 + r_348; - if (t_24 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + for (int r_306 = 0; r_306 < 16; r_306++) { + int t_17 = 16 + r_306; + if (t_17 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6439,41 +6104,41 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_25 = 32 + r_349; - if (t_25 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + for (int r_307 = 0; r_307 < 16; r_307++) { + int t_18 = 32 + r_307; + if (t_18 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6621,41 +6286,41 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_26 = 48 + r_350; - if (t_26 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + for (int r_308 = 0; r_308 < 16; r_308++) { + int t_19 = 48 + r_308; + if (t_19 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;");