From 12750d1cbd7c38e4e8577d81bda5bab9da05b942 Mon Sep 17 00:00:00 2001 From: Yingyi Huang Date: Thu, 5 Nov 2026 09:35:28 -0800 Subject: [PATCH 1/5] fix(cake_kda_det): gate-gradient accuracy of the fused-l2n GLM intra stage (diagonal TF32 rounding) Follow-up to #7817. The fused-l2n GLM intra programs (glm_intra_l2n on sm_100a / sm_107a, glm_intra_v30_l2n on sm_103a) rounded the two members of each diagonal pair term of the gate gradient on different TF32 values, so dg / dA_log / dt_bias lost accuracy at fast gate decay. Both members now use the same TF32 operands and cancel to FP32 rounding; no MMA is added and dq / dbeta keep their bits. Record names and selection are unchanged; the two programs are renamed with their new builds. --- .../cake_kda_det_train/build.json | 204 +- .../cake_kda_det_train/cake_jit.py | 8 +- ...a_det_train_37ed4f705c286adadd40_kernel.cu | 909 ++-- ...a_det_train_54786ffd01bd9b6bb416_kernel.cu | 1301 +++-- ...a_det_train_6308ef19a5bf0691fcfe_kernel.cu | 4822 ++++++++--------- ...det_train_a135a53a639f33f921ab_binding.cu} | 16 +- ..._det_train_a135a53a639f33f921ab_kernel.cu} | 1415 +++-- ...a_det_train_aa6b5c23c97e1cb9880a_kernel.cu | 1293 +++-- ...det_train_ad4c93dbb0435933c8a1_binding.cu} | 16 +- ..._det_train_ad4c93dbb0435933c8a1_kernel.cu} | 2606 +++++---- ...a_det_train_b16a0c1729083004fe80_kernel.cu | 3193 +++++------ ...a_det_train_c27b09eb5f5881be2969_kernel.cu | 1321 +++-- ...a_det_train_ea14e16d1fc71364ba03_kernel.cu | 4071 +++++++------- 13 files changed, 10839 insertions(+), 10336 deletions(-) rename flashinfer/experimental/cake_kda_det_train/csrc/{cake_kda_det_train_328afe43951979e798d1_binding.cu => cake_kda_det_train_a135a53a639f33f921ab_binding.cu} (98%) rename flashinfer/experimental/cake_kda_det_train/csrc/{cake_kda_det_train_9ad80f98f22e0b3bce08_kernel.cu => cake_kda_det_train_a135a53a639f33f921ab_kernel.cu} (90%) rename flashinfer/experimental/cake_kda_det_train/csrc/{cake_kda_det_train_9ad80f98f22e0b3bce08_binding.cu => cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu} (98%) rename flashinfer/experimental/cake_kda_det_train/csrc/{cake_kda_det_train_328afe43951979e798d1_kernel.cu => cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu} (85%) diff --git a/flashinfer/experimental/cake_kda_det_train/build.json b/flashinfer/experimental/cake_kda_det_train/build.json index 20f13b8c718..4f32c1f7a93 100644 --- a/flashinfer/experimental/cake_kda_det_train/build.json +++ b/flashinfer/experimental/cake_kda_det_train/build.json @@ -55,9 +55,9 @@ } }, "generator": { - "cake_revision": "3462767011d1e0de9c84de41bbb90397fa1ebafb", + "cake_revision": "66dfdbba90b203e45aa6f05fb399f056cf525863", "cuda": "13.4.92", - "rendered_at": "2026-11-04T08:37:59Z" + "rendered_at": "2026-11-05T07:57:50Z" }, "libraries": [ { @@ -1362,69 +1362,6 @@ "csrc/cake_kda_det_train_3162ed758e5ddfb97c17_binding.cu" ] }, - { - "arch": "sm_100a", - "closure_sha256": "d67635fb51a6027711e2f5ce62e5b0989cd9209e4c40f2c87e39f26d9ef72cc5", - "entry": "run", - "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", - "name": "cake_kda_det_train_328afe43951979e798d1_sm_100a", - "nvcc_flags": [ - "--use_fast_math" - ], - "nvcc_flags_if": [], - "path": "aot/cake_kda_det_train_328afe43951979e798d1_sm_100a.so", - "sha256": { - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu": "a2280dbb0317696d0de63cfcc1f058d72f078434209069346fe7226985ca1a4f", - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu": "6280fa5f6602c176fa9302f54db505440868f4646457059095c217bdb73a5c74" - }, - "sources": [ - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu", - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu" - ] - }, - { - "arch": "sm_103a", - "closure_sha256": "5f6a438c7ab175156bc9b6904cf18003c88d9f7ece6e9ccf0ca270d96de27d39", - "entry": "run", - "gencode": "-gencode=arch=compute_103a,code=sm_103a", - "min_cuda": "12.9", - "name": "cake_kda_det_train_328afe43951979e798d1_sm_103a", - "nvcc_flags": [ - "--use_fast_math" - ], - "nvcc_flags_if": [], - "path": "aot/cake_kda_det_train_328afe43951979e798d1_sm_103a.so", - "sha256": { - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu": "a2280dbb0317696d0de63cfcc1f058d72f078434209069346fe7226985ca1a4f", - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu": "6280fa5f6602c176fa9302f54db505440868f4646457059095c217bdb73a5c74" - }, - "sources": [ - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu", - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu" - ] - }, - { - "arch": "sm_107a", - "closure_sha256": "22287ac196b49a8a2197bc12e46031243498844c97cfb992c3698d1f406f97c9", - "entry": "run", - "gencode": "-gencode=arch=compute_107a,code=sm_107a", - "min_cuda": "13.4", - "name": "cake_kda_det_train_328afe43951979e798d1_sm_107a", - "nvcc_flags": [ - "--use_fast_math" - ], - "nvcc_flags_if": [], - "path": "aot/cake_kda_det_train_328afe43951979e798d1_sm_107a.so", - "sha256": { - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu": "a2280dbb0317696d0de63cfcc1f058d72f078434209069346fe7226985ca1a4f", - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu": "6280fa5f6602c176fa9302f54db505440868f4646457059095c217bdb73a5c74" - }, - "sources": [ - "csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu", - "csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu" - ] - }, { "arch": "sm_100a", "closure_sha256": "fb9a5b5ed6a3a643c231bff18c0274bae18ad4d1a309773bebc176decb9997bb", @@ -1490,7 +1427,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "b92a651f508dcd7fcb456899a611dee82b426afc34ce35d00b0367bcb90478e1", + "closure_sha256": "690c6d305fa5613747ebc52a8fb7b73a214d7afb2f1f64d82e7a02dc4ca6a1d2", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1502,7 +1439,7 @@ "path": "aot/cake_kda_det_train_37ed4f705c286adadd40_sm_100a.so", "sha256": { "csrc/cake_kda_det_train_37ed4f705c286adadd40_binding.cu": "5202f3619a6ff62aafe1a71c3cdef4d087ee3980a6dbc17eaf6c786fdc3863af", - "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "bb39cb550bf2117abedd25f0b15f341d7cb5d3d7a773a15b75990284e1c949d1" + "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "5e333a7df6481cbce675eff02a232727b5cfcf243413f9951a31f46e15c21aa5" }, "sources": [ "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu", @@ -1511,7 +1448,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "a9cc65744f8f52c242100d916a0aa15b95a541cda910fe406030b7731f24b248", + "closure_sha256": "079af35e3f5d255d74875bdb7e15cb689f0ab701c720dc74ddb449630ea112b7", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -1523,7 +1460,7 @@ "path": "aot/cake_kda_det_train_37ed4f705c286adadd40_sm_103a.so", "sha256": { "csrc/cake_kda_det_train_37ed4f705c286adadd40_binding.cu": "5202f3619a6ff62aafe1a71c3cdef4d087ee3980a6dbc17eaf6c786fdc3863af", - "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "bb39cb550bf2117abedd25f0b15f341d7cb5d3d7a773a15b75990284e1c949d1" + "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "5e333a7df6481cbce675eff02a232727b5cfcf243413f9951a31f46e15c21aa5" }, "sources": [ "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu", @@ -1532,7 +1469,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "06a0b4e168e5f7cd47a1ffcb025998be5820d28735b4237acb5b907ef0d35ccc", + "closure_sha256": "b2481688ed2c449f22d38ba7dde8dab602ee01e9c5dbdcaeb19a8a989e82065d", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -1544,7 +1481,7 @@ "path": "aot/cake_kda_det_train_37ed4f705c286adadd40_sm_107a.so", "sha256": { "csrc/cake_kda_det_train_37ed4f705c286adadd40_binding.cu": "5202f3619a6ff62aafe1a71c3cdef4d087ee3980a6dbc17eaf6c786fdc3863af", - "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "bb39cb550bf2117abedd25f0b15f341d7cb5d3d7a773a15b75990284e1c949d1" + "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu": "5e333a7df6481cbce675eff02a232727b5cfcf243413f9951a31f46e15c21aa5" }, "sources": [ "csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu", @@ -2078,7 +2015,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "3609084337ae5d9a546310b69bb2fb9fb834e1a485d1703015703957c312f1e1", + "closure_sha256": "c5cd38315e7557fa15c790fbbc886457ab7103527e2619c3a77d953d4dc22422", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -2090,7 +2027,7 @@ "path": "aot/cake_kda_det_train_54786ffd01bd9b6bb416_sm_100a.so", "sha256": { "csrc/cake_kda_det_train_54786ffd01bd9b6bb416_binding.cu": "5ce7e802c6b0993c88a0902e1e01a19510157db4b3af37238eb4f07ef7cff3bf", - "csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu": "f4c9f7508fd777c791df07e6a872ec959ea0efa4c655a64eec155092001e1491" + "csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu": "ec6ddd0feb25279017baa490321b21987af12940f8886b9f0c8c77326488b6dd" }, "sources": [ "csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu", @@ -2351,7 +2288,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "588999d7dd184af482d50c8dcc2bbe11ad4e63d3d8d5412575b1d8753a29e323", + "closure_sha256": "173b5af1082b4ee723c7515b4de34b3c33c2be0a4c9ac4815e3c32f763581452", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -2363,7 +2300,7 @@ "path": "aot/cake_kda_det_train_6308ef19a5bf0691fcfe_sm_100a.so", "sha256": { "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_binding.cu": "443d9bd452639c361d50f955da2c34a0127adac99eefecf25fd3ee876f614cf4", - "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "e8d8e01623afc65774a41e7a6dfe45993b15f6b27a11d01c27179c76b5b5db7a" + "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "03331aeb5c9eec8b79f370946cdf424e7405a4644c9ac87afb8d23932749f70b" }, "sources": [ "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu", @@ -2372,7 +2309,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "584e6cc20076b734cb00e9b8b5013ce66bd6a60780520c6eec61ffcee87354ad", + "closure_sha256": "ff0a8f4669a4732c15b65a97d42c004cae2cabca76ac8cbb93b11bf94d223f8d", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -2384,7 +2321,7 @@ "path": "aot/cake_kda_det_train_6308ef19a5bf0691fcfe_sm_103a.so", "sha256": { "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_binding.cu": "443d9bd452639c361d50f955da2c34a0127adac99eefecf25fd3ee876f614cf4", - "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "e8d8e01623afc65774a41e7a6dfe45993b15f6b27a11d01c27179c76b5b5db7a" + "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "03331aeb5c9eec8b79f370946cdf424e7405a4644c9ac87afb8d23932749f70b" }, "sources": [ "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu", @@ -2393,7 +2330,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "ddfee41a698e7899e10c3ea88bd0240c46f72f51acfb8e80b1f35662e9db6b4b", + "closure_sha256": "8bbd0bacf61e59235f1efe3b6017f8246374d82c81b5511bb291dd402d751149", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -2405,7 +2342,7 @@ "path": "aot/cake_kda_det_train_6308ef19a5bf0691fcfe_sm_107a.so", "sha256": { "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_binding.cu": "443d9bd452639c361d50f955da2c34a0127adac99eefecf25fd3ee876f614cf4", - "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "e8d8e01623afc65774a41e7a6dfe45993b15f6b27a11d01c27179c76b5b5db7a" + "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu": "03331aeb5c9eec8b79f370946cdf424e7405a4644c9ac87afb8d23932749f70b" }, "sources": [ "csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu", @@ -3800,23 +3737,23 @@ }, { "arch": "sm_103a", - "closure_sha256": "28ccc6594e50f48ab26dde04d911c06fbf31988a14c65b54a863284097d3aecc", + "closure_sha256": "543e22ef8c1b9f331328cccba1a27d95f33255cfcd1ff13343a0b0406356f00a", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_kda_det_train_9ad80f98f22e0b3bce08_sm_103a", + "name": "cake_kda_det_train_a135a53a639f33f921ab_sm_103a", "nvcc_flags": [ "--use_fast_math" ], "nvcc_flags_if": [], - "path": "aot/cake_kda_det_train_9ad80f98f22e0b3bce08_sm_103a.so", + "path": "aot/cake_kda_det_train_a135a53a639f33f921ab_sm_103a.so", "sha256": { - "csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_binding.cu": "f93536d86b23d345a48d1061d31ab92fdf34358df70f0556c636f5e607a1cd2b", - "csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_kernel.cu": "85eec7d912d6a8512f86898f7653bfe532642223c57276612937fb2a3fdf9588" + "csrc/cake_kda_det_train_a135a53a639f33f921ab_binding.cu": "dd0fd6333712a0d413c21180afc4d1c5e67eecb746f2764b27430256944cfd20", + "csrc/cake_kda_det_train_a135a53a639f33f921ab_kernel.cu": "76603465d1aec9fc51afd4cf116acbcbb2d7f74da58d421b26ba276405b8ec62" }, "sources": [ - "csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_kernel.cu", - "csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_binding.cu" + "csrc/cake_kda_det_train_a135a53a639f33f921ab_kernel.cu", + "csrc/cake_kda_det_train_a135a53a639f33f921ab_binding.cu" ] }, { @@ -4010,7 +3947,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "774a1be99b4d9cace2b218eac26278924aa81ab1250a605caef0778fc1f30f5d", + "closure_sha256": "8f3c67b0b027fe7f712268d6a2cfef75990908ea9cff633093a1d2187bdf6b76", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4022,7 +3959,7 @@ "path": "aot/cake_kda_det_train_aa6b5c23c97e1cb9880a_sm_107a.so", "sha256": { "csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_binding.cu": "debc670b1148d75a7a7a5921f82594f94e87c6a0e3164c8d96e78d72f40109a1", - "csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu": "298916e699ec308a3f6f915e493b8fa4fc32773ed05b570ac8309d559922eb28" + "csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu": "fb9a573e5e64c79c62009e3d468b443e61dd36ab4f5b3f010e80835f01718ebf" }, "sources": [ "csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu", @@ -4092,6 +4029,69 @@ "csrc/cake_kda_det_train_ac974e171d56fc3ce10b_binding.cu" ] }, + { + "arch": "sm_100a", + "closure_sha256": "b829f5bb6544edac23cd576e8fd10299eaa224a9a1a99c498f4da58a5a0aa7a3", + "entry": "run", + "gencode": "-gencode=arch=compute_100a,code=sm_100a", + "min_cuda": "12.8", + "name": "cake_kda_det_train_ad4c93dbb0435933c8a1_sm_100a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_kda_det_train_ad4c93dbb0435933c8a1_sm_100a.so", + "sha256": { + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu": "d764d721a4dee7c1711eb31a083621008300ecca4d848e6605f52382ef19860d", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu": "89771fb71e1e717a0a76dc466148d1d1ae551911872e5c22f2446694f780190d" + }, + "sources": [ + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu" + ] + }, + { + "arch": "sm_103a", + "closure_sha256": "b4888425f74e8c7feea2070856dd688f91ba57b2473a13ab093b0df48867f403", + "entry": "run", + "gencode": "-gencode=arch=compute_103a,code=sm_103a", + "min_cuda": "12.9", + "name": "cake_kda_det_train_ad4c93dbb0435933c8a1_sm_103a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_kda_det_train_ad4c93dbb0435933c8a1_sm_103a.so", + "sha256": { + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu": "d764d721a4dee7c1711eb31a083621008300ecca4d848e6605f52382ef19860d", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu": "89771fb71e1e717a0a76dc466148d1d1ae551911872e5c22f2446694f780190d" + }, + "sources": [ + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "0a6eb50f3f56ad769b58ea39f24e31ac71de1d0e48823c22d5df425f45d6ec9c", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_kda_det_train_ad4c93dbb0435933c8a1_sm_107a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_kda_det_train_ad4c93dbb0435933c8a1_sm_107a.so", + "sha256": { + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu": "d764d721a4dee7c1711eb31a083621008300ecca4d848e6605f52382ef19860d", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu": "89771fb71e1e717a0a76dc466148d1d1ae551911872e5c22f2446694f780190d" + }, + "sources": [ + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu", + "csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu" + ] + }, { "arch": "sm_100a", "closure_sha256": "fedf5e76ae0bd20e07ba2faafa288acd00f225fdd7f8c4ea3274cd2f2b2710fc", @@ -4157,7 +4157,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "e53c26da2f9396fb00c28e0b1026af4b6bf7eb15b26510621698ff24e8562a5f", + "closure_sha256": "a96205205c8b3f8c81383cb0b2a05ac0c8df59a6665fd14a15ec557639ef6fb7", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -4169,7 +4169,7 @@ "path": "aot/cake_kda_det_train_b16a0c1729083004fe80_sm_100a.so", "sha256": { "csrc/cake_kda_det_train_b16a0c1729083004fe80_binding.cu": "794bfda5c0f38f682e0e47f7cbe493ba7259e8e3452cbf57e7ae8418774bd19c", - "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "97168c857930c9c0b958c53c9fc454e22f3db5466d9e6662605b4365c7e7373b" + "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "0a7461ee10c6fc594a467a600d7db7440f2bcaa32ceee0defacf08fb67695d5c" }, "sources": [ "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu", @@ -4178,7 +4178,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "73dc6915cf91d10ee0f7cf64edb39915c25a5f718b42068332cdf87495812157", + "closure_sha256": "c580e4dc184c0e79fa5d3da8d3c78e5b1c230acce2bfdfac73a44daa66458c42", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -4190,7 +4190,7 @@ "path": "aot/cake_kda_det_train_b16a0c1729083004fe80_sm_103a.so", "sha256": { "csrc/cake_kda_det_train_b16a0c1729083004fe80_binding.cu": "794bfda5c0f38f682e0e47f7cbe493ba7259e8e3452cbf57e7ae8418774bd19c", - "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "97168c857930c9c0b958c53c9fc454e22f3db5466d9e6662605b4365c7e7373b" + "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "0a7461ee10c6fc594a467a600d7db7440f2bcaa32ceee0defacf08fb67695d5c" }, "sources": [ "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu", @@ -4199,7 +4199,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "6c4d32517a7f05553af5004f8de4be2ecc7c36406d99645e4f33920d79245a4d", + "closure_sha256": "8fcf6c93f1fae0283afe853ae3aa670854c39cd542c853339037098de5d9e934", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4211,7 +4211,7 @@ "path": "aot/cake_kda_det_train_b16a0c1729083004fe80_sm_107a.so", "sha256": { "csrc/cake_kda_det_train_b16a0c1729083004fe80_binding.cu": "794bfda5c0f38f682e0e47f7cbe493ba7259e8e3452cbf57e7ae8418774bd19c", - "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "97168c857930c9c0b958c53c9fc454e22f3db5466d9e6662605b4365c7e7373b" + "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu": "0a7461ee10c6fc594a467a600d7db7440f2bcaa32ceee0defacf08fb67695d5c" }, "sources": [ "csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu", @@ -4346,7 +4346,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "f47aad030860d78edaab9dc981a976aa4eacf304d6f575a12916910ae9e367d5", + "closure_sha256": "d3f05c11435bea7db383e301a1fd650f028a37a6b72cc37230234e04eae2b2c9", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -4358,7 +4358,7 @@ "path": "aot/cake_kda_det_train_c27b09eb5f5881be2969_sm_103a.so", "sha256": { "csrc/cake_kda_det_train_c27b09eb5f5881be2969_binding.cu": "d9d3b2beefd5de36931ceef887eab30fc410aa5a7de2179528b93dbb32992b5a", - "csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu": "b4c04f2c996190ba3f750aa6193ddcbc0c89a560da3c324f06cbaa223bd0169e" + "csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu": "97de6f775d8944105d6690adce2ac575693d3c7eb7b704a556a95e4b1659030f" }, "sources": [ "csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu", @@ -5186,7 +5186,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "a5f2c4044a4aceacd6d92955aa224bcd6463933cd8db00955969090ef63736c5", + "closure_sha256": "ccde9d8566ae05a6a9679a92c3671c0f31dea64236c85e247213e23e78564134", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -5198,7 +5198,7 @@ "path": "aot/cake_kda_det_train_ea14e16d1fc71364ba03_sm_100a.so", "sha256": { "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_binding.cu": "252a5ddb33acb0fc2a321354f365d5dd61560c0852e5c673f3b5c3a2fa181782", - "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "a18014739476a9a46b0357eb2cef236b99058427cf3dd8aeb58f0f85adb82951" + "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "376a2d7b861c534f2b41f5748cd4fab0f8993d8b21c0c645949f8859ce11c498" }, "sources": [ "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu", @@ -5207,7 +5207,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "4129ba9d2edb5707676fea6497f4f5e0cc844ccab4788eb4d919fb6290c4f6d8", + "closure_sha256": "2e5a195be353b3251ead917e0fc7db239c4326907b64f8ee1281cbf7e69e2357", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -5219,7 +5219,7 @@ "path": "aot/cake_kda_det_train_ea14e16d1fc71364ba03_sm_103a.so", "sha256": { "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_binding.cu": "252a5ddb33acb0fc2a321354f365d5dd61560c0852e5c673f3b5c3a2fa181782", - "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "a18014739476a9a46b0357eb2cef236b99058427cf3dd8aeb58f0f85adb82951" + "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "376a2d7b861c534f2b41f5748cd4fab0f8993d8b21c0c645949f8859ce11c498" }, "sources": [ "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu", @@ -5228,7 +5228,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "4645d75bb22547eeb8e78f7aada81a738f63103ee460296d859b3315e055d291", + "closure_sha256": "e83e9937787c1620b61ee3ca1500045b6762880ef3aa7d18b9e06b449150ba0e", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -5240,7 +5240,7 @@ "path": "aot/cake_kda_det_train_ea14e16d1fc71364ba03_sm_107a.so", "sha256": { "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_binding.cu": "252a5ddb33acb0fc2a321354f365d5dd61560c0852e5c673f3b5c3a2fa181782", - "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "a18014739476a9a46b0357eb2cef236b99058427cf3dd8aeb58f0f85adb82951" + "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu": "376a2d7b861c534f2b41f5748cd4fab0f8993d8b21c0c645949f8859ce11c498" }, "sources": [ "csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu", diff --git a/flashinfer/experimental/cake_kda_det_train/cake_jit.py b/flashinfer/experimental/cake_kda_det_train/cake_jit.py index 21e7ad9bd04..8e12a828e10 100644 --- a/flashinfer/experimental/cake_kda_det_train/cake_jit.py +++ b/flashinfer/experimental/cake_kda_det_train/cake_jit.py @@ -7366,7 +7366,7 @@ ], "ffi_entry": "run", "grid_rule": {"kind": "chunks", "x_factor": 1}, - "kernel_symbol": "kernel_cake_kda_det_train_328afe43951979e798d1", + "kernel_symbol": "kernel_cake_kda_det_train_ad4c93dbb0435933c8a1", "l2n": ["fused"], "launch": { "block": (288, 1, 1), @@ -7375,7 +7375,7 @@ "dynamic_smem_bytes": 115200, "use_pdl": False, }, - "library": "cake_kda_det_train_328afe43951979e798d1", + "library": "cake_kda_det_train_ad4c93dbb0435933c8a1", "logical": { "beta": "beta_int", "chunk_bos": "chunk_bos", @@ -7457,7 +7457,7 @@ ], "ffi_entry": "run", "grid_rule": {"kind": "chunks", "x_factor": 1}, - "kernel_symbol": "kernel_cake_kda_det_train_9ad80f98f22e0b3bce08", + "kernel_symbol": "kernel_cake_kda_det_train_a135a53a639f33f921ab", "l2n": ["fused"], "launch": { "block": (288, 1, 1), @@ -7466,7 +7466,7 @@ "dynamic_smem_bytes": 115200, "use_pdl": False, }, - "library": "cake_kda_det_train_9ad80f98f22e0b3bce08", + "library": "cake_kda_det_train_a135a53a639f33f921ab", "logical": { "beta": "beta_int", "chunk_bos": "chunk_bos", diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu index 9db0e86d2f0..0239f524cb4 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_37ed4f705c286adadd40_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2249,12 +2272,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_3 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * e2_3); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_3, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_3 : __fmaf_rn(dl, _rcp_170, e2_3)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2278,12 +2307,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_4 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * e2_4); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_4, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_4, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_4 : __fmaf_rn(dl_1, _rcp_174, e2_4)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2307,12 +2342,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_5 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * e2_5); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_5, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_5, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_5 : __fmaf_rn(dl_3, _rcp_178, e2_5)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2336,12 +2377,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_6 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * e2_6); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_6, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_6, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_6 : __fmaf_rn(dl_5, _rcp_182, e2_6)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_2 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_1 = approx_exp2(dc2_2.x); float e2_1_1 = approx_exp2(dc2_2.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_190 = approx_rcp(k_t0_2); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_191 = approx_rcp(k_t1_2); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_2.x; pan_641[3] = p2_2.y; int t0_3 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_2 = approx_exp2(dc2_3.x); float e2_1_2 = approx_exp2(dc2_3.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_194 = approx_rcp(k_t0_3); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_195 = approx_rcp(k_t1_3); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_3.x; pan_641[5] = p2_3.y; int t0_4 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(_f2_61, make_float2(g_t0_4, g_t1_4)); float e2_0_3 = approx_exp2(dc2_4.x); float e2_1_3 = approx_exp2(dc2_4.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_198 = approx_rcp(k_t0_4); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_199 = approx_rcp(k_t1_4); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_4.x; pan_641[7] = p2_4.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2505,12 +2576,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_202 = approx_rcp(k_t0_5); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_203 = approx_rcp(k_t1_5); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_5.x; pan_703[1] = p2_5.y; int t0_6 = 8 * th + 2; @@ -2534,12 +2611,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_2.x); float en_1_1 = approx_exp2(dn2_2.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_206 = approx_rcp(k_t0_6); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_207 = approx_rcp(k_t1_6); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_6.x; pan_703[3] = p2_6.y; int t0_7 = 8 * th + 4; @@ -2563,12 +2646,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_3.x); float en_1_2 = approx_exp2(dn2_3.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_210 = approx_rcp(k_t0_7); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_211 = approx_rcp(k_t1_7); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_7.x; pan_703[5] = p2_7.y; int t0_8 = 8 * th + 6; @@ -2592,12 +2681,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_4.x); float en_1_3 = approx_exp2(dn2_4.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_214 = approx_rcp(k_t0_8); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_215 = approx_rcp(k_t1_8); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_8.x; pan_703[7] = p2_8.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2639,12 +2734,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_5.x); float en_1_4 = approx_exp2(dn2_5.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_218 = approx_rcp(k_t0_9); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_219 = approx_rcp(k_t1_9); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_9.x; pan_777[1] = p2_9.y; int t0_10 = 16 + 8 * th + 2; @@ -2668,12 +2769,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_6.x); float en_1_5 = approx_exp2(dn2_6.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_222 = approx_rcp(k_t0_10); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_223 = approx_rcp(k_t1_10); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_10.x; pan_777[3] = p2_10.y; int t0_11 = 16 + 8 * th + 4; @@ -2697,12 +2804,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_7.x); float en_1_6 = approx_exp2(dn2_7.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_226 = approx_rcp(k_t0_11); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_227 = approx_rcp(k_t1_11); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_11.x; pan_777[5] = p2_11.y; int t0_12 = 16 + 8 * th + 6; @@ -2726,12 +2839,18 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 dn2_8 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_8.x); float en_1_7 = approx_exp2(dn2_8.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_230 = approx_rcp(k_t0_12); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_231 = approx_rcp(k_t1_12); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_12.x; pan_777[7] = p2_12.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2802,37 +2921,37 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_4 = approx_exp2(d2_57.x); float e1_4 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_4, e1_4); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_4, e1_4); float2 v2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_4.x; v_562_0[1] = v2_4.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_5 = approx_exp2(d2_58.x); float e1_5 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_5, e1_5); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_5, e1_5); float2 v2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_5.x; v_562_0[3] = v2_5.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_6 = approx_exp2(d2_59.x); float e1_6 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_6, e1_6); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_6, e1_6); float2 v2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_6.x; v_562_0[5] = v2_6.y; float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_7 = approx_exp2(d2_60.x); float e1_7 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_7, e1_7); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_7, e1_7); float2 v2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_7.x; v_562_0[7] = v2_7.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2966,7 +3085,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -2998,7 +3122,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3030,7 +3159,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_2)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3062,7 +3196,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_3)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3094,7 +3233,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_4)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3126,7 +3270,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_5)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3158,7 +3307,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_6)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3190,7 +3344,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_7)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3205,31 +3364,31 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_216 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_216; + float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_248; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_217 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_217; + float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_249; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_218 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_218; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_250; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_219 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_219; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_251; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_220 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_220; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_252; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_221 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_221; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_253; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_222 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_222; - float _shfl_xor_sync_223 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_223; - float _shfl_xor_sync_224 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_224; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_254; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_255; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_256; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); if (32 + 8 * th < clen) { @@ -3337,7 +3496,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; + float qe_7 = q_t_1_8 * e_dg_0; + float kp_19 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_0; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_14[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_0) * _tmem_load_15[7]); + float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_16[7], k_t_0, cr_8)) + dgf_0; s_run_1059[0] = s_run_1059[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3369,7 +3533,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[6]; - float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, (dk2b_2 - _tmem_load_16[6]) * k_t_2) + dgf_2; + float qe_8 = q_t_1_9 * e_dg_2; + float kp_20 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_14[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_2) * _tmem_load_15[6]); + float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_16[6], k_t_2, cr_9)) + dgf_2; s_run_1059[0] = s_run_1059[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3401,7 +3570,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[5]; - float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, (dk2b_3 - _tmem_load_16[5]) * k_t_3) + dgf_3; + float qe_9 = q_t_1_10 * e_dg_3; + float kp_21 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_3; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_14[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_3) * _tmem_load_15[5]); + float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_16[5], k_t_3, cr_10)) + dgf_3; s_run_1059[0] = s_run_1059[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3433,7 +3607,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[4]; - float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, (dk2b_4 - _tmem_load_16[4]) * k_t_4) + dgf_4; + float qe_10 = q_t_1_11 * e_dg_4; + float kp_22 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_4; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_14[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_4) * _tmem_load_15[4]); + float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_16[4], k_t_4, cr_11)) + dgf_4; s_run_1059[0] = s_run_1059[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3465,7 +3644,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[3]; - float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, (dk2b_5 - _tmem_load_16[3]) * k_t_5) + dgf_5; + float qe_11 = q_t_1_12 * e_dg_5; + float kp_23 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_5; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_14[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_5) * _tmem_load_15[3]); + float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_16[3], k_t_5, cr_12)) + dgf_5; s_run_1059[0] = s_run_1059[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3497,7 +3681,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[2]; - float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, (dk2b_6 - _tmem_load_16[2]) * k_t_6) + dgf_6; + float qe_12 = q_t_1_13 * e_dg_6; + float kp_24 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_6; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_14[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_6) * _tmem_load_15[2]); + float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_16[2], k_t_6, cr_13)) + dgf_6; s_run_1059[0] = s_run_1059[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3529,7 +3718,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[1]; - float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, (dk2b_7 - _tmem_load_16[1]) * k_t_7) + dgf_7; + float qe_13 = q_t_1_14 * e_dg_7; + float kp_25 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_7; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_14[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_7) * _tmem_load_15[1]); + float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_16[1], k_t_7, cr_14)) + dgf_7; s_run_1059[0] = s_run_1059[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3561,7 +3755,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_16[0]; - float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, (dk2b_8 - _tmem_load_16[0]) * k_t_8) + dgf_8; + float qe_14 = q_t_1_15 * e_dg_8; + float kp_26 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_8; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_14[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_8) * _tmem_load_15[0]); + float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_16[0], k_t_8, cr_15)) + dgf_8; s_run_1059[0] = s_run_1059[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3576,31 +3775,31 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int b2_1231 = lane_1 >> 2 & 1; float v4_1232[4]; float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_225 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_225; + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_257; float send_2 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_226 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_226; + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_258; float send_3 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_227 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_227; + float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_259; float send_4 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_228 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_228; + float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_260; float v2_1237[2]; float send_5 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_229 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_229; + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_261; float send_6 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_230 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_230; + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); + v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_262; float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_231 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_231; - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_232; - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_233; + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); + float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_263; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); + v1_1241 = v1_1241 + _shfl_xor_sync_264; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); + v1_1241 = v1_1241 + _shfl_xor_sync_265; asm volatile("tcgen05.fence::before_thread_sync;"); if (48 + 8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + (48 + 8 * th)) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(98304 - 32768 * th >> 2) + d_lane]; @@ -3652,8 +3851,8 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM } mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_234 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_234), "f"(tot_1228) : "memory"); + uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_1228) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3682,9 +3881,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_0 = offset + _tmem_load_17[0]; if (t_9 < clen) { { - auto _gst_235 = dg_base + goff_1_0; - __builtin_assume(__isGlobal(_gst_235)); - *(reinterpret_cast(_gst_235) + (0)) = dg_1_0; + auto _gst_267 = dg_base + goff_1_0; + __builtin_assume(__isGlobal(_gst_267)); + *(reinterpret_cast(_gst_267) + (0)) = dg_1_0; } } int t_10 = 48 + 8 * th + 1; @@ -3692,9 +3891,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_1 = offset + _tmem_load_17[1]; if (t_10 < clen) { { - auto _gst_236 = dg_base + goff_1_1; - __builtin_assume(__isGlobal(_gst_236)); - *(reinterpret_cast(_gst_236) + (0)) = dg_1_1; + auto _gst_268 = dg_base + goff_1_1; + __builtin_assume(__isGlobal(_gst_268)); + *(reinterpret_cast(_gst_268) + (0)) = dg_1_1; } } int t_11 = 48 + 8 * th + 2; @@ -3702,9 +3901,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_2 = offset + _tmem_load_17[2]; if (t_11 < clen) { { - auto _gst_237 = dg_base + goff_1_2; - __builtin_assume(__isGlobal(_gst_237)); - *(reinterpret_cast(_gst_237) + (0)) = dg_1_2; + auto _gst_269 = dg_base + goff_1_2; + __builtin_assume(__isGlobal(_gst_269)); + *(reinterpret_cast(_gst_269) + (0)) = dg_1_2; } } int t_12 = 48 + 8 * th + 3; @@ -3712,9 +3911,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_3 = offset + _tmem_load_17[3]; if (t_12 < clen) { { - auto _gst_238 = dg_base + goff_1_3; - __builtin_assume(__isGlobal(_gst_238)); - *(reinterpret_cast(_gst_238) + (0)) = dg_1_3; + auto _gst_270 = dg_base + goff_1_3; + __builtin_assume(__isGlobal(_gst_270)); + *(reinterpret_cast(_gst_270) + (0)) = dg_1_3; } } int t_13 = 48 + 8 * th + 4; @@ -3722,9 +3921,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_4 = offset + _tmem_load_17[4]; if (t_13 < clen) { { - auto _gst_239 = dg_base + goff_1_4; - __builtin_assume(__isGlobal(_gst_239)); - *(reinterpret_cast(_gst_239) + (0)) = dg_1_4; + auto _gst_271 = dg_base + goff_1_4; + __builtin_assume(__isGlobal(_gst_271)); + *(reinterpret_cast(_gst_271) + (0)) = dg_1_4; } } int t_14 = 48 + 8 * th + 5; @@ -3732,9 +3931,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_5 = offset + _tmem_load_17[5]; if (t_14 < clen) { { - auto _gst_240 = dg_base + goff_1_5; - __builtin_assume(__isGlobal(_gst_240)); - *(reinterpret_cast(_gst_240) + (0)) = dg_1_5; + auto _gst_272 = dg_base + goff_1_5; + __builtin_assume(__isGlobal(_gst_272)); + *(reinterpret_cast(_gst_272) + (0)) = dg_1_5; } } int t_15 = 48 + 8 * th + 6; @@ -3742,9 +3941,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_6 = offset + _tmem_load_17[6]; if (t_15 < clen) { { - auto _gst_241 = dg_base + goff_1_6; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast(_gst_241) + (0)) = dg_1_6; + auto _gst_273 = dg_base + goff_1_6; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast(_gst_273) + (0)) = dg_1_6; } } int t_16 = 48 + 8 * th + 7; @@ -3752,9 +3951,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1_7 = offset + _tmem_load_17[7]; if (t_16 < clen) { { - auto _gst_242 = dg_base + goff_1_7; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast(_gst_242) + (0)) = dg_1_7; + auto _gst_274 = dg_base + goff_1_7; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast(_gst_274) + (0)) = dg_1_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3772,9 +3971,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_0 = offset_1250 + _tmem_load_19[0]; if (t_17 < clen) { { - auto _gst_243 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast(_gst_243) + (0)) = dg_0; + auto _gst_275 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast(_gst_275) + (0)) = dg_0; } } int t_18 = 32 + 8 * th + 1; @@ -3782,9 +3981,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_1 = offset_1250 + _tmem_load_19[1]; if (t_18 < clen) { { - auto _gst_244 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast(_gst_244) + (0)) = dg_1; + auto _gst_276 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast(_gst_276) + (0)) = dg_1; } } int t_19 = 32 + 8 * th + 2; @@ -3792,9 +3991,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_2 = offset_1250 + _tmem_load_19[2]; if (t_19 < clen) { { - auto _gst_245 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast(_gst_245) + (0)) = dg_2; + auto _gst_277 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast(_gst_277) + (0)) = dg_2; } } int t_20 = 32 + 8 * th + 3; @@ -3802,9 +4001,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_3 = offset_1250 + _tmem_load_19[3]; if (t_20 < clen) { { - auto _gst_246 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast(_gst_246) + (0)) = dg_3; + auto _gst_278 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast(_gst_278) + (0)) = dg_3; } } int t_21 = 32 + 8 * th + 4; @@ -3812,9 +4011,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_4 = offset_1250 + _tmem_load_19[4]; if (t_21 < clen) { { - auto _gst_247 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast(_gst_247) + (0)) = dg_4; + auto _gst_279 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast(_gst_279) + (0)) = dg_4; } } int t_22 = 32 + 8 * th + 5; @@ -3822,9 +4021,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_5 = offset_1250 + _tmem_load_19[5]; if (t_22 < clen) { { - auto _gst_248 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast(_gst_248) + (0)) = dg_5; + auto _gst_280 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast(_gst_280) + (0)) = dg_5; } } int t_23 = 32 + 8 * th + 6; @@ -3832,9 +4031,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_6 = offset_1250 + _tmem_load_19[6]; if (t_23 < clen) { { - auto _gst_249 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast(_gst_249) + (0)) = dg_6; + auto _gst_281 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast(_gst_281) + (0)) = dg_6; } } int t_24 = 32 + 8 * th + 7; @@ -3842,15 +4041,15 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_7 = offset_1250 + _tmem_load_19[7]; if (t_24 < clen) { { - auto _gst_250 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast(_gst_250) + (0)) = dg_7; + auto _gst_282 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast(_gst_282) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi_1248 + t_lo_1249; { - uint32_t _addr_251 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_251), "f"(off_acc[0]) : "memory"); + uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3910,7 +4109,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[7]; - float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, (dk2b_9 - _tmem_load_24[7]) * k_t_9) + dgf_9; + float qe_15 = q_t_1_16 * e_dg_9; + float kp_27 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_9; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_9) * _tmem_load_23[7]); + float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_24[7], k_t_9, cr_16)) + dgf_9; s_run_1302[0] = s_run_1302[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3942,7 +4146,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[6]; - float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, (dk2b_10 - _tmem_load_24[6]) * k_t_10) + dgf_10; + float qe_16 = q_t_1_17 * e_dg_10; + float kp_28 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_10; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_10) * _tmem_load_23[6]); + float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_24[6], k_t_10, cr_17)) + dgf_10; s_run_1302[0] = s_run_1302[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3974,7 +4183,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[5]; - float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, (dk2b_11 - _tmem_load_24[5]) * k_t_11) + dgf_11; + float qe_17 = q_t_1_18 * e_dg_11; + float kp_29 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_11; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_11) * _tmem_load_23[5]); + float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_24[5], k_t_11, cr_18)) + dgf_11; s_run_1302[0] = s_run_1302[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4006,7 +4220,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[4]; - float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, (dk2b_12 - _tmem_load_24[4]) * k_t_12) + dgf_12; + float qe_18 = q_t_1_19 * e_dg_12; + float kp_30 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_12; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_12) * _tmem_load_23[4]); + float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_24[4], k_t_12, cr_19)) + dgf_12; s_run_1302[0] = s_run_1302[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4038,7 +4257,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[3]; - float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, (dk2b_13 - _tmem_load_24[3]) * k_t_13) + dgf_13; + float qe_19 = q_t_1_20 * e_dg_13; + float kp_31 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_13; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_13) * _tmem_load_23[3]); + float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_24[3], k_t_13, cr_20)) + dgf_13; s_run_1302[0] = s_run_1302[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4070,7 +4294,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[2]; - float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, (dk2b_14 - _tmem_load_24[2]) * k_t_14) + dgf_14; + float qe_20 = q_t_1_21 * e_dg_14; + float kp_32 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_14; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_14) * _tmem_load_23[2]); + float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_24[2], k_t_14, cr_21)) + dgf_14; s_run_1302[0] = s_run_1302[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4102,7 +4331,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[1]; - float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, (dk2b_15 - _tmem_load_24[1]) * k_t_15) + dgf_15; + float qe_21 = q_t_1_22 * e_dg_15; + float kp_33 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_15; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_15) * _tmem_load_23[1]); + float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_24[1], k_t_15, cr_22)) + dgf_15; s_run_1302[0] = s_run_1302[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4134,7 +4368,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_24[0]; - float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, (dk2b_16 - _tmem_load_24[0]) * k_t_16) + dgf_16; + float qe_22 = q_t_1_23 * e_dg_16; + float kp_34 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_16; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_16) * _tmem_load_23[0]); + float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, __fmaf_rn(dk2b_16 - _tmem_load_24[0], k_t_16, cr_23)) + dgf_16; s_run_1302[0] = s_run_1302[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4149,31 +4388,31 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; float send_7 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_252; + float _shfl_xor_sync_284 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_284; float send_8 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_253; + float _shfl_xor_sync_285 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_285; float send_9 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_254; + float _shfl_xor_sync_286 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_286; float send_10 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_255; + float _shfl_xor_sync_287 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_287; float v2_1480[2]; float send_11 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_256; + float _shfl_xor_sync_288 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_288; float send_12 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_289; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_258; - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_259; - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_260; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_290; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_291; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_292; asm volatile("tcgen05.fence::before_thread_sync;"); if (16 + 8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + (16 + 8 * th)) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(81920 - 32768 * th >> 2) + d_lane]; @@ -4226,8 +4465,8 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_261 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_261), "f"(tot_1471) : "memory"); + uint32_t _addr_293 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_293), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4256,9 +4495,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_8 = offset_1488 + _tmem_load_25[0]; if (t_33 < clen) { { - auto _gst_262 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_262)); - *(reinterpret_cast(_gst_262) + (0)) = dg_8; + auto _gst_294 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_294)); + *(reinterpret_cast(_gst_294) + (0)) = dg_8; } } int t_34 = 16 + 8 * th + 1; @@ -4266,9 +4505,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_9 = offset_1488 + _tmem_load_25[1]; if (t_34 < clen) { { - auto _gst_263 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_263)); - *(reinterpret_cast(_gst_263) + (0)) = dg_9; + auto _gst_295 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_295)); + *(reinterpret_cast(_gst_295) + (0)) = dg_9; } } int t_35 = 16 + 8 * th + 2; @@ -4276,9 +4515,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_10 = offset_1488 + _tmem_load_25[2]; if (t_35 < clen) { { - auto _gst_264 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_264)); - *(reinterpret_cast(_gst_264) + (0)) = dg_10; + auto _gst_296 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_296)); + *(reinterpret_cast(_gst_296) + (0)) = dg_10; } } int t_36 = 16 + 8 * th + 3; @@ -4286,9 +4525,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_11 = offset_1488 + _tmem_load_25[3]; if (t_36 < clen) { { - auto _gst_265 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_265)); - *(reinterpret_cast(_gst_265) + (0)) = dg_11; + auto _gst_297 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_297)); + *(reinterpret_cast(_gst_297) + (0)) = dg_11; } } int t_37 = 16 + 8 * th + 4; @@ -4296,9 +4535,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_12 = offset_1488 + _tmem_load_25[4]; if (t_37 < clen) { { - auto _gst_266 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_266)); - *(reinterpret_cast(_gst_266) + (0)) = dg_12; + auto _gst_298 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_298)); + *(reinterpret_cast(_gst_298) + (0)) = dg_12; } } int t_38 = 16 + 8 * th + 5; @@ -4306,9 +4545,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_13 = offset_1488 + _tmem_load_25[5]; if (t_38 < clen) { { - auto _gst_267 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_13; + auto _gst_299 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_13; } } int t_39 = 16 + 8 * th + 6; @@ -4316,9 +4555,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_14 = offset_1488 + _tmem_load_25[6]; if (t_39 < clen) { { - auto _gst_268 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_14; + auto _gst_300 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_14; } } int t_40 = 16 + 8 * th + 7; @@ -4326,15 +4565,15 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_15 = offset_1488 + _tmem_load_25[7]; if (t_40 < clen) { { - auto _gst_269 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_15; + auto _gst_301 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_1486 + t_lo_1487; { - uint32_t _addr_270 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_270), "f"(off_acc[0]) : "memory"); + uint32_t _addr_302 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_302), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4388,7 +4627,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[7]; - float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, (dk2b_17 - _tmem_load_28[7]) * k_t_17) + dgf_17; + float qe_23 = q_t_1_24 * e_dg_17; + float kp_35 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_35; + float kbe_23 = (float)kb_23 * e_dg_17; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_35 * e_dg_17) * _tmem_load_27[7]); + float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, __fmaf_rn(dk2b_17 - _tmem_load_28[7], k_t_17, cr_24)) + dgf_17; s_run_1524[0] = s_run_1524[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4420,7 +4664,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[6]; - float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, (dk2b_18 - _tmem_load_28[6]) * k_t_18) + dgf_18; + float qe_24 = q_t_1_25 * e_dg_18; + float kp_36 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_36; + float kbe_24 = (float)kb_24 * e_dg_18; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_36 * e_dg_18) * _tmem_load_27[6]); + float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, __fmaf_rn(dk2b_18 - _tmem_load_28[6], k_t_18, cr_25)) + dgf_18; s_run_1524[0] = s_run_1524[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4452,7 +4701,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[5]; - float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, (dk2b_19 - _tmem_load_28[5]) * k_t_19) + dgf_19; + float qe_25 = q_t_1_26 * e_dg_19; + float kp_37 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_37; + float kbe_25 = (float)kb_25 * e_dg_19; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_37 * e_dg_19) * _tmem_load_27[5]); + float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, __fmaf_rn(dk2b_19 - _tmem_load_28[5], k_t_19, cr_26)) + dgf_19; s_run_1524[0] = s_run_1524[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4484,7 +4738,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[4]; - float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, (dk2b_20 - _tmem_load_28[4]) * k_t_20) + dgf_20; + float qe_26 = q_t_1_27 * e_dg_20; + float kp_38 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_38; + float kbe_26 = (float)kb_26 * e_dg_20; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_38 * e_dg_20) * _tmem_load_27[4]); + float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, __fmaf_rn(dk2b_20 - _tmem_load_28[4], k_t_20, cr_27)) + dgf_20; s_run_1524[0] = s_run_1524[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4516,7 +4775,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[3]; - float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, (dk2b_21 - _tmem_load_28[3]) * k_t_21) + dgf_21; + float qe_27 = q_t_1_28 * e_dg_21; + float kp_39 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_39; + float kbe_27 = (float)kb_27 * e_dg_21; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_39 * e_dg_21) * _tmem_load_27[3]); + float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, __fmaf_rn(dk2b_21 - _tmem_load_28[3], k_t_21, cr_28)) + dgf_21; s_run_1524[0] = s_run_1524[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4548,7 +4812,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[2]; - float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, (dk2b_22 - _tmem_load_28[2]) * k_t_22) + dgf_22; + float qe_28 = q_t_1_29 * e_dg_22; + float kp_40 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_40; + float kbe_28 = (float)kb_28 * e_dg_22; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_40 * e_dg_22) * _tmem_load_27[2]); + float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, __fmaf_rn(dk2b_22 - _tmem_load_28[2], k_t_22, cr_29)) + dgf_22; s_run_1524[0] = s_run_1524[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4580,7 +4849,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[1]; - float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, (dk2b_23 - _tmem_load_28[1]) * k_t_23) + dgf_23; + float qe_29 = q_t_1_30 * e_dg_23; + float kp_41 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_41; + float kbe_29 = (float)kb_29 * e_dg_23; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_41 * e_dg_23) * _tmem_load_27[1]); + float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, __fmaf_rn(dk2b_23 - _tmem_load_28[1], k_t_23, cr_30)) + dgf_23; s_run_1524[0] = s_run_1524[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4612,7 +4886,12 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + _tmem_load_28[0]; - float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, (dk2b_24 - _tmem_load_28[0]) * k_t_24) + dgf_24; + float qe_30 = q_t_1_31 * e_dg_24; + float kp_42 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_42; + float kbe_30 = (float)kb_30 * e_dg_24; + float cr_31 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_42 * e_dg_24) * _tmem_load_27[0]); + float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, __fmaf_rn(dk2b_24 - _tmem_load_28[0], k_t_24, cr_31)) + dgf_24; s_run_1524[0] = s_run_1524[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4627,31 +4906,31 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; float send_13 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_271; + float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_303; float send_14 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_272; + float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_304; float send_15 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_273; + float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_305; float send_16 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_274; + float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_306; float v2_1702[2]; float send_17 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_275; + float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_307; float send_18 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_276; + float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_308; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_277; - float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_278; - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_279; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_309; + float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_310; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_311; if (8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + 8 * th) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(98304 - 32768 * th >> 2) + d_lane]; } @@ -4702,8 +4981,8 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM } asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_280 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_280), "f"(tot_1693) : "memory"); + uint32_t _addr_312 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_312), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4732,9 +5011,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_16 = offset_1710 + _tmem_load_29[0]; if (t_49 < clen) { { - auto _gst_281 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_16; + auto _gst_313 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_16; } } int t_50 = 8 * th + 1; @@ -4742,9 +5021,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_17 = offset_1710 + _tmem_load_29[1]; if (t_50 < clen) { { - auto _gst_282 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_17; + auto _gst_314 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_17; } } int t_51 = 8 * th + 2; @@ -4752,9 +5031,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_18 = offset_1710 + _tmem_load_29[2]; if (t_51 < clen) { { - auto _gst_283 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_283)); - *(reinterpret_cast(_gst_283) + (0)) = dg_18; + auto _gst_315 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_315)); + *(reinterpret_cast(_gst_315) + (0)) = dg_18; } } int t_52 = 8 * th + 3; @@ -4762,9 +5041,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_19 = offset_1710 + _tmem_load_29[3]; if (t_52 < clen) { { - auto _gst_284 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast(_gst_284) + (0)) = dg_19; + auto _gst_316 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = dg_19; } } int t_53 = 8 * th + 4; @@ -4772,9 +5051,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_20 = offset_1710 + _tmem_load_29[4]; if (t_53 < clen) { { - auto _gst_285 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast(_gst_285) + (0)) = dg_20; + auto _gst_317 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast(_gst_317) + (0)) = dg_20; } } int t_54 = 8 * th + 5; @@ -4782,9 +5061,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_21 = offset_1710 + _tmem_load_29[5]; if (t_54 < clen) { { - auto _gst_286 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast(_gst_286) + (0)) = dg_21; + auto _gst_318 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = dg_21; } } int t_55 = 8 * th + 6; @@ -4792,9 +5071,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_22 = offset_1710 + _tmem_load_29[6]; if (t_55 < clen) { { - auto _gst_287 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast(_gst_287) + (0)) = dg_22; + auto _gst_319 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast(_gst_319) + (0)) = dg_22; } } int t_56 = 8 * th + 7; @@ -4802,9 +5081,9 @@ kernel_cake_kda_det_train_37ed4f705c286adadd40(const __grid_constant__ CUtensorM float dg_23 = offset_1710 + _tmem_load_29[7]; if (t_56 < clen) { { - auto _gst_288 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast(_gst_288) + (0)) = dg_23; + auto _gst_320 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1708 + t_lo_1709; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu index 3f76831b164..cc293b1672b 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_54786ffd01bd9b6bb416_kernel.cu @@ -130,6 +130,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -172,6 +179,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2235,15 +2258,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2264,15 +2293,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2293,15 +2328,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2322,15 +2363,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2366,12 +2413,18 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2391,12 +2444,18 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2416,12 +2475,18 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2441,12 +2506,18 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2491,15 +2562,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2520,15 +2597,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2549,15 +2632,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2578,15 +2667,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2625,15 +2720,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2654,15 +2755,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2683,15 +2790,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2712,15 +2825,21 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2791,37 +2910,37 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2833,29 +2952,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2863,29 +2982,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2953,24 +3072,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; - float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0) + dgf_0; + float qe = q_t0_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr)) + dgf_0; int goff_0 = t_0 * rs_g; int qoff_0 = t_0 * qrs; { - auto _gst_216 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast(_gst_216) + (0)) = dg_o_0; + auto _gst_248 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast(_gst_248) + (0)) = dg_o_0; } if (t_0 < clen) { { - auto _gst_217 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_249 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_218 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_250 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -2996,24 +3120,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_1 = dk2_1 * bt_t_1; float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; - float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1) + dgf_1; + float qe_0 = q_t0_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0)) + dgf_1; int goff_1 = t_1 * rs_g; int qoff_1 = t_1 * qrs; { - auto _gst_219 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast(_gst_219) + (0)) = dg_o_1; + auto _gst_251 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast(_gst_251) + (0)) = dg_o_1; } if (t_1 < clen) { { - auto _gst_220 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_252 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_221 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_253 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3039,24 +3168,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; - float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2) + dgf_2; + float qe_1 = q_t0_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1)) + dgf_2; int goff_2 = t_2 * rs_g; int qoff_2 = t_2 * qrs; { - auto _gst_222 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast(_gst_222) + (0)) = dg_o_2; + auto _gst_254 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast(_gst_254) + (0)) = dg_o_2; } if (t_2 < clen) { { - auto _gst_223 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_255 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_224 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_256 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3082,24 +3216,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; - float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3) + dgf_3; + float qe_2 = q_t0_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2)) + dgf_3; int goff_3 = t_3 * rs_g; int qoff_3 = t_3 * qrs; { - auto _gst_225 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast(_gst_225) + (0)) = dg_o_3; + auto _gst_257 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast(_gst_257) + (0)) = dg_o_3; } if (t_3 < clen) { { - auto _gst_226 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_258 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_227 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_259 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3125,24 +3264,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; - float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4) + dgf_4; + float qe_3 = q_t0_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3)) + dgf_4; int goff_4 = t_4 * rs_g; int qoff_4 = t_4 * qrs; { - auto _gst_228 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast(_gst_228) + (0)) = dg_o_4; + auto _gst_260 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast(_gst_260) + (0)) = dg_o_4; } if (t_4 < clen) { { - auto _gst_229 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_261 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_230 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_262 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3168,24 +3312,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; - float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5) + dgf_5; + float qe_4 = q_t0_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4)) + dgf_5; int goff_5 = t_5 * rs_g; int qoff_5 = t_5 * qrs; { - auto _gst_231 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast(_gst_231) + (0)) = dg_o_5; + auto _gst_263 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast(_gst_263) + (0)) = dg_o_5; } if (t_5 < clen) { { - auto _gst_232 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_232)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_232) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_264 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_264)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_264) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_233 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_233)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_233) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_265 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_265)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3211,24 +3360,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; - float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6) + dgf_6; + float qe_5 = q_t0_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5)) + dgf_6; int goff_6 = t_6 * rs_g; int qoff_6 = t_6 * qrs; { - auto _gst_234 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_234)); - *(reinterpret_cast(_gst_234) + (0)) = dg_o_6; + auto _gst_266 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_266)); + *(reinterpret_cast(_gst_266) + (0)) = dg_o_6; } if (t_6 < clen) { { - auto _gst_235 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_235)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_235) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_267 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_267)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_267) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_236 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_236)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_236) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_268 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_268)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3254,24 +3408,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; - float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7) + dgf_7; + float qe_6 = q_t0_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6)) + dgf_7; int goff_7 = t_7 * rs_g; int qoff_7 = t_7 * qrs; { - auto _gst_237 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_237)); - *(reinterpret_cast(_gst_237) + (0)) = dg_o_7; + auto _gst_269 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_269)); + *(reinterpret_cast(_gst_269) + (0)) = dg_o_7; } if (t_7 < clen) { { - auto _gst_238 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_238)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_238) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_270 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_270)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_270) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_239 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_239)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_239) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_271 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_271)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dk_o_7; } } int b4 = lane_1 >> 4 & 1; @@ -3279,31 +3438,31 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_240; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_272; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_241 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_241; + float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_273; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_242 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_242; + float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_274; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_243 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_243; + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_275; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_244 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_244; + float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_276; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_245 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_245; + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_277; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_246 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_246; - float _shfl_xor_sync_247 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_247; - float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_248; + float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_278; + float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_279; + float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_280; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3361,24 +3520,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; - float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8) + dgf_8; + float qe_7 = q_t0_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7)) + dgf_8; int goff_8 = t_8 * rs_g; int qoff_8 = t_8 * qrs; { - auto _gst_249 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast(_gst_249) + (0)) = dg_o_8; + auto _gst_281 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast(_gst_281) + (0)) = dg_o_8; } if (t_8 < clen) { { - auto _gst_250 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_282 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_251 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_283 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3404,24 +3568,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; - float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9) + dgf_9; + float qe_8 = q_t0_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8)) + dgf_9; int goff_9 = t_9 * rs_g; int qoff_9 = t_9 * qrs; { - auto _gst_252 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast(_gst_252) + (0)) = dg_o_9; + auto _gst_284 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast(_gst_284) + (0)) = dg_o_9; } if (t_9 < clen) { { - auto _gst_253 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_285 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_254 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_286 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3447,24 +3616,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; - float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10) + dgf_10; + float qe_9 = q_t0_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9)) + dgf_10; int goff_10 = t_10 * rs_g; int qoff_10 = t_10 * qrs; { - auto _gst_255 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast(_gst_255) + (0)) = dg_o_10; + auto _gst_287 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast(_gst_287) + (0)) = dg_o_10; } if (t_10 < clen) { { - auto _gst_256 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_288 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_257 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_257)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_289 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_289)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3490,24 +3664,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; - float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11) + dgf_11; + float qe_10 = q_t0_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10)) + dgf_11; int goff_11 = t_11 * rs_g; int qoff_11 = t_11 * qrs; { - auto _gst_258 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_258)); - *(reinterpret_cast(_gst_258) + (0)) = dg_o_11; + auto _gst_290 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_290)); + *(reinterpret_cast(_gst_290) + (0)) = dg_o_11; } if (t_11 < clen) { { - auto _gst_259 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_259)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_291 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_291)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_260 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_260)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_292 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_292)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3533,24 +3712,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; - float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12) + dgf_12; + float qe_11 = q_t0_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11)) + dgf_12; int goff_12 = t_12 * rs_g; int qoff_12 = t_12 * qrs; { - auto _gst_261 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_261)); - *(reinterpret_cast(_gst_261) + (0)) = dg_o_12; + auto _gst_293 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_293)); + *(reinterpret_cast(_gst_293) + (0)) = dg_o_12; } if (t_12 < clen) { { - auto _gst_262 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_262)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_294 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_294)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_263 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_263)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_295 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_295)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3576,24 +3760,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; - float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13) + dgf_13; + float qe_12 = q_t0_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12)) + dgf_13; int goff_13 = t_13 * rs_g; int qoff_13 = t_13 * qrs; { - auto _gst_264 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_264)); - *(reinterpret_cast(_gst_264) + (0)) = dg_o_13; + auto _gst_296 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_296)); + *(reinterpret_cast(_gst_296) + (0)) = dg_o_13; } if (t_13 < clen) { { - auto _gst_265 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_265)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_297 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_297)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_266 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_266)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_266) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_298 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_298)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3619,24 +3808,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; - float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14) + dgf_14; + float qe_13 = q_t0_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13)) + dgf_14; int goff_14 = t_14 * rs_g; int qoff_14 = t_14 * qrs; { - auto _gst_267 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_o_14; + auto _gst_299 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_o_14; } if (t_14 < clen) { { - auto _gst_268 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_300 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_300) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_269 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_269) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_301 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3662,24 +3856,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; - float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15) + dgf_15; + float qe_14 = q_t0_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14)) + dgf_15; int goff_15 = t_15 * rs_g; int qoff_15 = t_15 * qrs; { - auto _gst_270 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_o_15; + auto _gst_302 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_o_15; } if (t_15 < clen) { { - auto _gst_271 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_303 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_303) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_272 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_272) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_304 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dk_o_15; } } int b4_0 = lane_1 >> 4 & 1; @@ -3687,31 +3886,31 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_273; + float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_305; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_274; + float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_306; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_275; + float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_307; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_276; + float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_308; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_277; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_309; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_278; + float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_310; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_279; - float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_280; - float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_281; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_311; + float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_312; + float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_313; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); mbarrier_wait(p1b_ready_addr, 0); @@ -3770,24 +3969,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; - float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16) + dgf_16; + float qe_15 = q_t0_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15)) + dgf_16; int goff_16 = t_16 * rs_g; int qoff_16 = t_16 * qrs; { - auto _gst_282 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_o_16; + auto _gst_314 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_o_16; } if (t_16 < clen) { { - auto _gst_283 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_283)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_315 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_315)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_315) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_284 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_316 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_17 = 16 + 8 * th + 1; @@ -3813,24 +4017,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; - float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17) + dgf_17; + float qe_16 = q_t0_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16)) + dgf_17; int goff_17 = t_17 * rs_g; int qoff_17 = t_17 * qrs; { - auto _gst_285 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast(_gst_285) + (0)) = dg_o_17; + auto _gst_317 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast(_gst_317) + (0)) = dg_o_17; } if (t_17 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_18 = 16 + 8 * th + 2; @@ -3856,24 +4065,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; - float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18) + dgf_18; + float qe_17 = q_t0_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17)) + dgf_18; int goff_18 = t_18 * rs_g; int qoff_18 = t_18 * qrs; { - auto _gst_288 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast(_gst_288) + (0)) = dg_o_18; + auto _gst_320 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = dg_o_18; } if (t_18 < clen) { { - auto _gst_289 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_321 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_290 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_322 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_19 = 16 + 8 * th + 3; @@ -3899,24 +4113,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; - float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19) + dgf_19; + float qe_18 = q_t0_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18)) + dgf_19; int goff_19 = t_19 * rs_g; int qoff_19 = t_19 * qrs; { - auto _gst_291 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast(_gst_291) + (0)) = dg_o_19; + auto _gst_323 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast(_gst_323) + (0)) = dg_o_19; } if (t_19 < clen) { { - auto _gst_292 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_324 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_293 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_325 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_20 = 16 + 8 * th + 4; @@ -3942,24 +4161,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; - float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20) + dgf_20; + float qe_19 = q_t0_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19)) + dgf_20; int goff_20 = t_20 * rs_g; int qoff_20 = t_20 * qrs; { - auto _gst_294 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast(_gst_294) + (0)) = dg_o_20; + auto _gst_326 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast(_gst_326) + (0)) = dg_o_20; } if (t_20 < clen) { { - auto _gst_295 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_327 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_296 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_328 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_21 = 16 + 8 * th + 5; @@ -3985,24 +4209,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; - float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21) + dgf_21; + float qe_20 = q_t0_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20)) + dgf_21; int goff_21 = t_21 * rs_g; int qoff_21 = t_21 * qrs; { - auto _gst_297 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast(_gst_297) + (0)) = dg_o_21; + auto _gst_329 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast(_gst_329) + (0)) = dg_o_21; } if (t_21 < clen) { { - auto _gst_298 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_330 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_299 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_331 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_22 = 16 + 8 * th + 6; @@ -4028,24 +4257,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; - float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22) + dgf_22; + float qe_21 = q_t0_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21)) + dgf_22; int goff_22 = t_22 * rs_g; int qoff_22 = t_22 * qrs; { - auto _gst_300 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_300)); - *(reinterpret_cast(_gst_300) + (0)) = dg_o_22; + auto _gst_332 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_332)); + *(reinterpret_cast(_gst_332) + (0)) = dg_o_22; } if (t_22 < clen) { { - auto _gst_301 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_301)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_333 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_333)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_302 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_302)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_302) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_334 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_334)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_23 = 16 + 8 * th + 7; @@ -4071,24 +4305,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; - float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23) + dgf_23; + float qe_22 = q_t0_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22)) + dgf_23; int goff_23 = t_23 * rs_g; int qoff_23 = t_23 * qrs; { - auto _gst_303 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_303)); - *(reinterpret_cast(_gst_303) + (0)) = dg_o_23; + auto _gst_335 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_335)); + *(reinterpret_cast(_gst_335) + (0)) = dg_o_23; } if (t_23 < clen) { { - auto _gst_304 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_304)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_336 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_336)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_336) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_305 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_305)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_305) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_337 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_337)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dk_o_23; } } int b4_1 = lane_1 >> 4 & 1; @@ -4096,31 +4335,31 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_306; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_338; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_307; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_339; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_308; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_340; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_309; + float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_341; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_310; + float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_342; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_311; + float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_343; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_312; - float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_313; - float _shfl_xor_sync_314 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_314; + float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_344; + float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_345; + float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_346; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); mbarrier_wait(epi_full_addr + (th) * 8, 0); @@ -4173,24 +4412,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; - float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24) + dgf_24; + float qe_23 = q_t0_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23)) + dgf_24; int goff_24 = t_24 * rs_g; int qoff_24 = t_24 * qrs; { - auto _gst_315 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_o_24; + auto _gst_347 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_o_24; } if (t_24 < clen) { { - auto _gst_316 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_348 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_348) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_317 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_349 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_349) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_25 = 8 * th + 1; @@ -4216,24 +4460,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_25 = dk2_25 * bt_t_25; float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; - float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25) + dgf_25; + float qe_24 = q_t0_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24)) + dgf_25; int goff_25 = t_25 * rs_g; int qoff_25 = t_25 * qrs; { - auto _gst_318 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = dg_o_25; + auto _gst_350 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_350)); + *(reinterpret_cast(_gst_350) + (0)) = dg_o_25; } if (t_25 < clen) { { - auto _gst_319 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_351 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_320 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_352 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_26 = 8 * th + 2; @@ -4259,24 +4508,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_26 = dk2_26 * bt_t_26; float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; - float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26) + dgf_26; + float qe_25 = q_t0_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25)) + dgf_26; int goff_26 = t_26 * rs_g; int qoff_26 = t_26 * qrs; { - auto _gst_321 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast(_gst_321) + (0)) = dg_o_26; + auto _gst_353 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast(_gst_353) + (0)) = dg_o_26; } if (t_26 < clen) { { - auto _gst_322 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_354 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_323 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_355 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_27 = 8 * th + 3; @@ -4302,24 +4556,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_27 = dk2_27 * bt_t_27; float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; - float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27) + dgf_27; + float qe_26 = q_t0_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26)) + dgf_27; int goff_27 = t_27 * rs_g; int qoff_27 = t_27 * qrs; { - auto _gst_324 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast(_gst_324) + (0)) = dg_o_27; + auto _gst_356 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast(_gst_356) + (0)) = dg_o_27; } if (t_27 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_28 = 8 * th + 4; @@ -4345,24 +4604,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_28 = dk2_28 * bt_t_28; float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; - float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28) + dgf_28; + float qe_27 = q_t0_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27)) + dgf_28; int goff_28 = t_28 * rs_g; int qoff_28 = t_28 * qrs; { - auto _gst_327 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast(_gst_327) + (0)) = dg_o_28; + auto _gst_359 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast(_gst_359) + (0)) = dg_o_28; } if (t_28 < clen) { { - auto _gst_328 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_360 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_329 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_361 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_29 = 8 * th + 5; @@ -4388,24 +4652,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_29 = dk2_29 * bt_t_29; float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; - float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29) + dgf_29; + float qe_28 = q_t0_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28)) + dgf_29; int goff_29 = t_29 * rs_g; int qoff_29 = t_29 * qrs; { - auto _gst_330 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast(_gst_330) + (0)) = dg_o_29; + auto _gst_362 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast(_gst_362) + (0)) = dg_o_29; } if (t_29 < clen) { { - auto _gst_331 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_363 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_332 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_364 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_30 = 8 * th + 6; @@ -4431,24 +4700,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_30 = dk2_30 * bt_t_30; float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; - float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30) + dgf_30; + float qe_29 = q_t0_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29)) + dgf_30; int goff_30 = t_30 * rs_g; int qoff_30 = t_30 * qrs; { - auto _gst_333 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast(_gst_333) + (0)) = dg_o_30; + auto _gst_365 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast(_gst_365) + (0)) = dg_o_30; } if (t_30 < clen) { { - auto _gst_334 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_366 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_335 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_335)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_335) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_367 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_367)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_367) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_31 = 8 * th + 7; @@ -4474,24 +4748,29 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM float dk2b_31 = dk2_31 * bt_t_31; float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; - float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31) + dgf_31; + float qe_30 = q_t0_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30)) + dgf_31; int goff_31 = t_31 * rs_g; int qoff_31 = t_31 * qrs; { - auto _gst_336 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_336)); - *(reinterpret_cast(_gst_336) + (0)) = dg_o_31; + auto _gst_368 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_368)); + *(reinterpret_cast(_gst_368) + (0)) = dg_o_31; } if (t_31 < clen) { { - auto _gst_337 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_337)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_369 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_369)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_369) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_338 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_338)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_338) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_370 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_370)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_370) + (0)) = (__nv_bfloat16)dk_o_31; } } int b4_2 = lane_1 >> 4 & 1; @@ -4499,31 +4778,31 @@ kernel_cake_kda_det_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorM int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_371; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_372; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_341; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_373; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_342; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_374; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_343; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_375; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_344; + float _shfl_xor_sync_376 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_376; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_345; - float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_346; - float _shfl_xor_sync_347 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_347; + float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_377; + float _shfl_xor_sync_378 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_378; + float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_379; if ((lane_1 & 3) == 0) { int e_star = lane_1 >> 2 & 7; pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_2; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu index c24eedc43de..7a39474948b 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_6308ef19a5bf0691fcfe_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -200,6 +201,21 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { } +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + __device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { float r; asm("mul.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); @@ -2281,12 +2297,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_0); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_0 : __fmaf_rn(dl, _rcp_170, e2_0)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2310,12 +2332,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_1); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_1, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_1, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_1 : __fmaf_rn(dl_1, _rcp_174, e2_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2339,12 +2367,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_2); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_2, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_2, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_2 : __fmaf_rn(dl_3, _rcp_178, e2_2)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2368,12 +2402,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_3); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_3, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_3, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_3 : __fmaf_rn(dl_5, _rcp_182, e2_3)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2409,12 +2449,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2434,12 +2480,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2459,12 +2511,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2484,12 +2542,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2537,12 +2601,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2566,12 +2636,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_1 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2595,12 +2671,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2624,12 +2706,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2671,12 +2759,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2700,12 +2794,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2729,12 +2829,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2758,12 +2864,18 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2834,37 +2946,37 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float2 d2_56 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_56.x); float e1_12 = approx_exp2(d2_56.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_57.x); float e1_13 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_58.x); float e1_14 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_59.x); float e1_15 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2876,29 +2988,29 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2906,29 +3018,29 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2998,7 +3110,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3030,7 +3147,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3062,7 +3184,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_1)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3094,7 +3221,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_2)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3126,7 +3258,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_3)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3158,7 +3295,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_4)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3190,7 +3332,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_5)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3222,7 +3369,12 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_6)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3237,31 +3389,31 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_216 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_216; + float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_248; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_217 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_217; + float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_249; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_218 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_218; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_250; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_219 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_219; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_251; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_220 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_220; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_252; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_221 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_221; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_253; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_222 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_222; - float _shfl_xor_sync_223 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_223; - float _shfl_xor_sync_224 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_224; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_254; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_255; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_256; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); int ew_0 = 98304 - 32768 * th >> 2; @@ -3271,8 +3423,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM for (int i = 0; i < 2; i++) { int j = qd + 4 * i; bool live = 32 + 8 * th + j < clen; - int _min_225 = ((32 + 8 * th + j) < (clen - 1) ? (32 + 8 * th + j) : (clen - 1)); - long long row = ext0 + (long long)_min_225; + int _min_257 = ((32 + 8 * th + j) < (clen - 1) ? (32 + 8 * th + j) : (clen - 1)); + long long row = ext0 + (long long)_min_257; float gq[4]; float gk_0[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" @@ -3289,26 +3441,26 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float rq[1]; float rk[1]; { - uint2 _vld_226; - _vld_226 = *reinterpret_cast(q_raw + (row * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_226 = reinterpret_cast(&_vld_226); + uint2 _vld_258; + _vld_258 = *reinterpret_cast(q_raw + (row * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_258 = reinterpret_cast(&_vld_258); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xq[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_226[_pair]) << 16); - (&xq[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_226[_pair]) & 0xffff0000u); + (&xq[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_258[_pair]) << 16); + (&xq[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_258[_pair]) & 0xffff0000u); } } { rq[0] = *reinterpret_cast(qk_rstd + ((0 + row) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_227; - _vld_227 = *reinterpret_cast(k_raw + (row * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_227 = reinterpret_cast(&_vld_227); + uint2 _vld_259; + _vld_259 = *reinterpret_cast(k_raw + (row * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_259 = reinterpret_cast(&_vld_259); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xk[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_227[_pair]) << 16); - (&xk[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_227[_pair]) & 0xffff0000u); + (&xk[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_259[_pair]) << 16); + (&xk[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_259[_pair]) & 0xffff0000u); } } { @@ -3318,106 +3470,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y[4]; float p[4]; y[0] = mul_f32_rn_noftz(xq[0], rq[0]); - float _fma_228; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_228) : "f"(gq[0]), "f"(y[0]), "f"(nz)); - p[0] = _fma_228; + float _fma_260; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_260) : "f"(gq[0]), "f"(y[0]), "f"(nz)); + p[0] = _fma_260; y[1] = mul_f32_rn_noftz(xq[1], rq[0]); - float _fma_229; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_229) : "f"(gq[1]), "f"(y[1]), "f"(nz)); - p[1] = _fma_229; + float _fma_261; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_261) : "f"(gq[1]), "f"(y[1]), "f"(nz)); + p[1] = _fma_261; y[2] = mul_f32_rn_noftz(xq[2], rq[0]); - float _fma_230; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_230) : "f"(gq[2]), "f"(y[2]), "f"(nz)); - p[2] = _fma_230; + float _fma_262; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_262) : "f"(gq[2]), "f"(y[2]), "f"(nz)); + p[2] = _fma_262; y[3] = mul_f32_rn_noftz(xq[3], rq[0]); - float _fma_231; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_231) : "f"(gq[3]), "f"(y[3]), "f"(nz)); - p[3] = _fma_231; - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, p[0], 4); - float _add_232; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_232) : "f"(p[0]), "f"(_shfl_xor_sync_233)); - p[0] = _add_232; - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, p[1], 4); - float _add_234; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_234) : "f"(p[1]), "f"(_shfl_xor_sync_235)); - p[1] = _add_234; - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, p[2], 4); - float _add_236; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_236) : "f"(p[2]), "f"(_shfl_xor_sync_237)); - p[2] = _add_236; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, p[3], 4); - float _add_238; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_238) : "f"(p[3]), "f"(_shfl_xor_sync_239)); - p[3] = _add_238; - float _shfl_xor_sync_241 = __shfl_xor_sync(0xFFFFFFFF, p[0], 2); - float _add_240; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_240) : "f"(p[0]), "f"(_shfl_xor_sync_241)); - p[0] = _add_240; - float _shfl_xor_sync_243 = __shfl_xor_sync(0xFFFFFFFF, p[1], 2); - float _add_242; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_242) : "f"(p[1]), "f"(_shfl_xor_sync_243)); - p[1] = _add_242; - float _shfl_xor_sync_245 = __shfl_xor_sync(0xFFFFFFFF, p[2], 2); - float _add_244; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_244) : "f"(p[2]), "f"(_shfl_xor_sync_245)); - p[2] = _add_244; - float _shfl_xor_sync_247 = __shfl_xor_sync(0xFFFFFFFF, p[3], 2); - float _add_246; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_246) : "f"(p[3]), "f"(_shfl_xor_sync_247)); - p[3] = _add_246; - float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, p[0], 1); - float _add_248; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_248) : "f"(p[0]), "f"(_shfl_xor_sync_249)); - p[0] = _add_248; - float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, p[1], 1); - float _add_250; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_250) : "f"(p[1]), "f"(_shfl_xor_sync_251)); - p[1] = _add_250; - float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, p[2], 1); - float _add_252; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_252) : "f"(p[2]), "f"(_shfl_xor_sync_253)); - p[2] = _add_252; - float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, p[3], 1); - float _add_254; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_254) : "f"(p[3]), "f"(_shfl_xor_sync_255)); - p[3] = _add_254; - float _add_256; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_256) : "f"(p[0]), "f"(p[2])); - float _add_257; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_257) : "f"(p[1]), "f"(p[3])); + float _fma_263; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_263) : "f"(gq[3]), "f"(y[3]), "f"(nz)); + p[3] = _fma_263; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, p[0], 4); + float _add_264; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_264) : "f"(p[0]), "f"(_shfl_xor_sync_265)); + p[0] = _add_264; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, p[1], 4); + float _add_266; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_266) : "f"(p[1]), "f"(_shfl_xor_sync_267)); + p[1] = _add_266; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, p[2], 4); + float _add_268; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_268) : "f"(p[2]), "f"(_shfl_xor_sync_269)); + p[2] = _add_268; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, p[3], 4); + float _add_270; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_270) : "f"(p[3]), "f"(_shfl_xor_sync_271)); + p[3] = _add_270; + float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, p[0], 2); + float _add_272; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_272) : "f"(p[0]), "f"(_shfl_xor_sync_273)); + p[0] = _add_272; + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, p[1], 2); + float _add_274; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_274) : "f"(p[1]), "f"(_shfl_xor_sync_275)); + p[1] = _add_274; + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, p[2], 2); + float _add_276; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_276) : "f"(p[2]), "f"(_shfl_xor_sync_277)); + p[2] = _add_276; + float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, p[3], 2); + float _add_278; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_278) : "f"(p[3]), "f"(_shfl_xor_sync_279)); + p[3] = _add_278; + float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, p[0], 1); + float _add_280; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_280) : "f"(p[0]), "f"(_shfl_xor_sync_281)); + p[0] = _add_280; + float _shfl_xor_sync_283 = __shfl_xor_sync(0xFFFFFFFF, p[1], 1); + float _add_282; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_282) : "f"(p[1]), "f"(_shfl_xor_sync_283)); + p[1] = _add_282; + float _shfl_xor_sync_285 = __shfl_xor_sync(0xFFFFFFFF, p[2], 1); + float _add_284; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_284) : "f"(p[2]), "f"(_shfl_xor_sync_285)); + p[2] = _add_284; + float _shfl_xor_sync_287 = __shfl_xor_sync(0xFFFFFFFF, p[3], 1); + float _add_286; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_286) : "f"(p[3]), "f"(_shfl_xor_sync_287)); + p[3] = _add_286; + float _add_288; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_288) : "f"(p[0]), "f"(p[2])); + float _add_289; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_289) : "f"(p[1]), "f"(p[3])); float sq; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq) : "f"(_add_256), "f"(_add_257)); - float _shfl_sync_260 = __shfl_sync(0xFFFFFFFF, sq, 0); - float _shfl_sync_261 = __shfl_sync(0xFFFFFFFF, sq, 8); - float _add_259; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_259) : "f"(_shfl_sync_260), "f"(_shfl_sync_261)); - float _shfl_sync_262 = __shfl_sync(0xFFFFFFFF, sq, 16); - float _add_258; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_258) : "f"(_add_259), "f"(_shfl_sync_262)); - float _shfl_sync_263 = __shfl_sync(0xFFFFFFFF, sq, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq) : "f"(_add_288), "f"(_add_289)); + float _shfl_sync_292 = __shfl_sync(0xFFFFFFFF, sq, 0); + float _shfl_sync_293 = __shfl_sync(0xFFFFFFFF, sq, 8); + float _add_291; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_291) : "f"(_shfl_sync_292), "f"(_shfl_sync_293)); + float _shfl_sync_294 = __shfl_sync(0xFFFFFFFF, sq, 16); + float _add_290; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_290) : "f"(_add_291), "f"(_shfl_sync_294)); + float _shfl_sync_295 = __shfl_sync(0xFFFFFFFF, sq, 24); float dot; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_258), "f"(_shfl_sync_263)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_290), "f"(_shfl_sync_295)); float o[4]; - float _fma_265; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_265) : "f"(y[0]), "f"(dot), "f"(nz)); - float _sub_264; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_264) : "f"(gq[0]), "f"(_fma_265)); - o[0] = mul_f32_rn_noftz(rq[0], _sub_264); - float _fma_267; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_267) : "f"(y[1]), "f"(dot), "f"(nz)); - float _sub_266; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_266) : "f"(gq[1]), "f"(_fma_267)); - o[1] = mul_f32_rn_noftz(rq[0], _sub_266); - float _fma_269; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_269) : "f"(y[2]), "f"(dot), "f"(nz)); - float _sub_268; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_268) : "f"(gq[2]), "f"(_fma_269)); - o[2] = mul_f32_rn_noftz(rq[0], _sub_268); - float _fma_271; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_271) : "f"(y[3]), "f"(dot), "f"(nz)); - float _sub_270; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_270) : "f"(gq[3]), "f"(_fma_271)); - o[3] = mul_f32_rn_noftz(rq[0], _sub_270); + float _fma_297; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_297) : "f"(y[0]), "f"(dot), "f"(nz)); + float _sub_296; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_296) : "f"(gq[0]), "f"(_fma_297)); + o[0] = mul_f32_rn_noftz(rq[0], _sub_296); + float _fma_299; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_299) : "f"(y[1]), "f"(dot), "f"(nz)); + float _sub_298; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_298) : "f"(gq[1]), "f"(_fma_299)); + o[1] = mul_f32_rn_noftz(rq[0], _sub_298); + float _fma_301; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_301) : "f"(y[2]), "f"(dot), "f"(nz)); + float _sub_300; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_300) : "f"(gq[2]), "f"(_fma_301)); + o[2] = mul_f32_rn_noftz(rq[0], _sub_300); + float _fma_303; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_303) : "f"(y[3]), "f"(dot), "f"(nz)); + float _sub_302; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_302) : "f"(gq[3]), "f"(_fma_303)); + o[3] = mul_f32_rn_noftz(rq[0], _sub_302); unsigned int ob[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -3428,106 +3580,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_0[4]; float p_0[4]; y_0[0] = mul_f32_rn_noftz(xk[0], rk[0]); - float _fma_272; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_272) : "f"(gk_0[0]), "f"(y_0[0]), "f"(nz)); - p_0[0] = _fma_272; + float _fma_304; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_304) : "f"(gk_0[0]), "f"(y_0[0]), "f"(nz)); + p_0[0] = _fma_304; y_0[1] = mul_f32_rn_noftz(xk[1], rk[0]); - float _fma_273; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_273) : "f"(gk_0[1]), "f"(y_0[1]), "f"(nz)); - p_0[1] = _fma_273; + float _fma_305; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_305) : "f"(gk_0[1]), "f"(y_0[1]), "f"(nz)); + p_0[1] = _fma_305; y_0[2] = mul_f32_rn_noftz(xk[2], rk[0]); - float _fma_274; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_274) : "f"(gk_0[2]), "f"(y_0[2]), "f"(nz)); - p_0[2] = _fma_274; + float _fma_306; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_306) : "f"(gk_0[2]), "f"(y_0[2]), "f"(nz)); + p_0[2] = _fma_306; y_0[3] = mul_f32_rn_noftz(xk[3], rk[0]); - float _fma_275; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_275) : "f"(gk_0[3]), "f"(y_0[3]), "f"(nz)); - p_0[3] = _fma_275; - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 4); - float _add_276; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_276) : "f"(p_0[0]), "f"(_shfl_xor_sync_277)); - p_0[0] = _add_276; - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 4); - float _add_278; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_278) : "f"(p_0[1]), "f"(_shfl_xor_sync_279)); - p_0[1] = _add_278; - float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 4); - float _add_280; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_280) : "f"(p_0[2]), "f"(_shfl_xor_sync_281)); - p_0[2] = _add_280; - float _shfl_xor_sync_283 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 4); - float _add_282; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_282) : "f"(p_0[3]), "f"(_shfl_xor_sync_283)); - p_0[3] = _add_282; - float _shfl_xor_sync_285 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 2); - float _add_284; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_284) : "f"(p_0[0]), "f"(_shfl_xor_sync_285)); - p_0[0] = _add_284; - float _shfl_xor_sync_287 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 2); - float _add_286; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_286) : "f"(p_0[1]), "f"(_shfl_xor_sync_287)); - p_0[1] = _add_286; - float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 2); - float _add_288; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_288) : "f"(p_0[2]), "f"(_shfl_xor_sync_289)); - p_0[2] = _add_288; - float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 2); - float _add_290; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_290) : "f"(p_0[3]), "f"(_shfl_xor_sync_291)); - p_0[3] = _add_290; - float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 1); - float _add_292; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_292) : "f"(p_0[0]), "f"(_shfl_xor_sync_293)); - p_0[0] = _add_292; - float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 1); - float _add_294; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_294) : "f"(p_0[1]), "f"(_shfl_xor_sync_295)); - p_0[1] = _add_294; - float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 1); - float _add_296; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_296) : "f"(p_0[2]), "f"(_shfl_xor_sync_297)); - p_0[2] = _add_296; - float _shfl_xor_sync_299 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 1); - float _add_298; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_298) : "f"(p_0[3]), "f"(_shfl_xor_sync_299)); - p_0[3] = _add_298; - float _add_300; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_300) : "f"(p_0[0]), "f"(p_0[2])); - float _add_301; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_301) : "f"(p_0[1]), "f"(p_0[3])); + float _fma_307; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_307) : "f"(gk_0[3]), "f"(y_0[3]), "f"(nz)); + p_0[3] = _fma_307; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 4); + float _add_308; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_308) : "f"(p_0[0]), "f"(_shfl_xor_sync_309)); + p_0[0] = _add_308; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 4); + float _add_310; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_310) : "f"(p_0[1]), "f"(_shfl_xor_sync_311)); + p_0[1] = _add_310; + float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 4); + float _add_312; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_312) : "f"(p_0[2]), "f"(_shfl_xor_sync_313)); + p_0[2] = _add_312; + float _shfl_xor_sync_315 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 4); + float _add_314; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_314) : "f"(p_0[3]), "f"(_shfl_xor_sync_315)); + p_0[3] = _add_314; + float _shfl_xor_sync_317 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 2); + float _add_316; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_316) : "f"(p_0[0]), "f"(_shfl_xor_sync_317)); + p_0[0] = _add_316; + float _shfl_xor_sync_319 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 2); + float _add_318; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_318) : "f"(p_0[1]), "f"(_shfl_xor_sync_319)); + p_0[1] = _add_318; + float _shfl_xor_sync_321 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 2); + float _add_320; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_320) : "f"(p_0[2]), "f"(_shfl_xor_sync_321)); + p_0[2] = _add_320; + float _shfl_xor_sync_323 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 2); + float _add_322; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_322) : "f"(p_0[3]), "f"(_shfl_xor_sync_323)); + p_0[3] = _add_322; + float _shfl_xor_sync_325 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 1); + float _add_324; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(p_0[0]), "f"(_shfl_xor_sync_325)); + p_0[0] = _add_324; + float _shfl_xor_sync_327 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 1); + float _add_326; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_326) : "f"(p_0[1]), "f"(_shfl_xor_sync_327)); + p_0[1] = _add_326; + float _shfl_xor_sync_329 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 1); + float _add_328; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(p_0[2]), "f"(_shfl_xor_sync_329)); + p_0[2] = _add_328; + float _shfl_xor_sync_331 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 1); + float _add_330; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(p_0[3]), "f"(_shfl_xor_sync_331)); + p_0[3] = _add_330; + float _add_332; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_332) : "f"(p_0[0]), "f"(p_0[2])); + float _add_333; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_333) : "f"(p_0[1]), "f"(p_0[3])); float sq_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_0) : "f"(_add_300), "f"(_add_301)); - float _shfl_sync_304 = __shfl_sync(0xFFFFFFFF, sq_0, 0); - float _shfl_sync_305 = __shfl_sync(0xFFFFFFFF, sq_0, 8); - float _add_303; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_303) : "f"(_shfl_sync_304), "f"(_shfl_sync_305)); - float _shfl_sync_306 = __shfl_sync(0xFFFFFFFF, sq_0, 16); - float _add_302; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_302) : "f"(_add_303), "f"(_shfl_sync_306)); - float _shfl_sync_307 = __shfl_sync(0xFFFFFFFF, sq_0, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_0) : "f"(_add_332), "f"(_add_333)); + float _shfl_sync_336 = __shfl_sync(0xFFFFFFFF, sq_0, 0); + float _shfl_sync_337 = __shfl_sync(0xFFFFFFFF, sq_0, 8); + float _add_335; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_335) : "f"(_shfl_sync_336), "f"(_shfl_sync_337)); + float _shfl_sync_338 = __shfl_sync(0xFFFFFFFF, sq_0, 16); + float _add_334; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(_add_335), "f"(_shfl_sync_338)); + float _shfl_sync_339 = __shfl_sync(0xFFFFFFFF, sq_0, 24); float dot_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_302), "f"(_shfl_sync_307)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_334), "f"(_shfl_sync_339)); float o_0[4]; - float _fma_309; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_309) : "f"(y_0[0]), "f"(dot_0), "f"(nz)); - float _sub_308; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_308) : "f"(gk_0[0]), "f"(_fma_309)); - o_0[0] = mul_f32_rn_noftz(rk[0], _sub_308); - float _fma_311; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_311) : "f"(y_0[1]), "f"(dot_0), "f"(nz)); - float _sub_310; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_310) : "f"(gk_0[1]), "f"(_fma_311)); - o_0[1] = mul_f32_rn_noftz(rk[0], _sub_310); - float _fma_313; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_313) : "f"(y_0[2]), "f"(dot_0), "f"(nz)); - float _sub_312; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_312) : "f"(gk_0[2]), "f"(_fma_313)); - o_0[2] = mul_f32_rn_noftz(rk[0], _sub_312); - float _fma_315; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_315) : "f"(y_0[3]), "f"(dot_0), "f"(nz)); - float _sub_314; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_314) : "f"(gk_0[3]), "f"(_fma_315)); - o_0[3] = mul_f32_rn_noftz(rk[0], _sub_314); + float _fma_341; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_341) : "f"(y_0[0]), "f"(dot_0), "f"(nz)); + float _sub_340; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_340) : "f"(gk_0[0]), "f"(_fma_341)); + o_0[0] = mul_f32_rn_noftz(rk[0], _sub_340); + float _fma_343; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_343) : "f"(y_0[1]), "f"(dot_0), "f"(nz)); + float _sub_342; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_342) : "f"(gk_0[1]), "f"(_fma_343)); + o_0[1] = mul_f32_rn_noftz(rk[0], _sub_342); + float _fma_345; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_345) : "f"(y_0[2]), "f"(dot_0), "f"(nz)); + float _sub_344; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_344) : "f"(gk_0[2]), "f"(_fma_345)); + o_0[2] = mul_f32_rn_noftz(rk[0], _sub_344); + float _fma_347; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_347) : "f"(y_0[3]), "f"(dot_0), "f"(nz)); + float _sub_346; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_346) : "f"(gk_0[3]), "f"(_fma_347)); + o_0[3] = mul_f32_rn_noftz(rk[0], _sub_346); unsigned int ob_0[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -3545,340 +3697,153 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float r_272[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1049 = r_272[0]; - float gc_1050 = gn_1049 - 21.640425f; - int eb_1051 = 98304 + th * -32768; - int ew_1052 = eb_1051 >> 2; - int qbase_1053 = pool_f32_addr + (unsigned int)(eb_1051 + 12288 + dh_lane * 1024); - int sb8_1054 = 48 + 8 * th; - int gb8_1055 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1054 * 128); - int kb8_1056 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1054 * 128); - float dbp_1057[8]; - float _tmem_load_12[8]; - tmem_ld_x8(&_tmem_load_12[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float _tmem_load_13[8]; - tmem_ld_x8(&_tmem_load_13[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float _tmem_load_14[8]; - tmem_ld_x8(&_tmem_load_14[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float _tmem_load_15[8]; - tmem_ld_x8(&_tmem_load_15[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float _tmem_load_16[8]; - tmem_ld_x8(&_tmem_load_16[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_1058[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[0])), "=r"(*reinterpret_cast(&btv_1058[(0) + 1])), "=r"(*reinterpret_cast(&btv_1058[(0) + 2])), "=r"(*reinterpret_cast(&btv_1058[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[4])), "=r"(*reinterpret_cast(&btv_1058[(4) + 1])), "=r"(*reinterpret_cast(&btv_1058[(4) + 2])), "=r"(*reinterpret_cast(&btv_1058[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); + float gn_0 = r_272[0]; + float gc_0 = gn_0 - 21.640425f; + int eb_0 = 98304 + th * -32768; + int ew_1 = eb_0 >> 2; + int qbase_0 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); + int sb8_0 = 48 + 8 * th; + int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); + int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); + int c_qa = 160 + 8 * th; + int c_ka = 176 + 8 * th; + int c_qd = 224 + 8 * th; + int c_kd = 240 + 8 * th; + int c_dk = 48 + 8 * th; + int c_bp = c_ka; + float s_run_0[1]; + s_run_0[0] = 0.0f; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j_1 = 7 - k; + float a_qd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j_1))); + float a_kd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd + j_1))); + float a_dk[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk + j_1))); + float a_qa[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa + j_1))); + float a_ka[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka + j_1))); + float btr[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_0 + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8_0 + j_1; + float r_273[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_273[0]; + unsigned int r_274[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_274[0] << 16); + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_275[0] << 16); + float q_m = ((t >= clen) ? 0.0f : q_t); + float dqf = pool_f32[ew_1 + 128 * j_1 + d_lane]; + float dkf = pool_f32[ew_1 + (1024 + 128 * j_1) + d_lane]; + float dgf = pool_f32[ew_1 + (2048 + 128 * j_1) + d_lane]; + float bt_t = btr[0]; + float e_dg = approx_exp2(g_t - gc_0); + float e_off = approx_exp2(g_t - gn_0); + float dq2 = __fmaf_rn(a_qa[0], e_off, a_qd[0] * e_dg); + float dk2 = __fmaf_rn(a_ka[0], e_off, a_kd[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + a_dk[0]; + float qe_7 = q_m * e_dg; + float kp_19 = k_t * bt_t; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, a_qd[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg) * a_kd[0]); + float dg_o = __fmaf_rn(q_m, dq2, __fmaf_rn(dk2b - a_dk[0], k_t, cr_7)) + dgf; + s_run_0[0] = s_run_0[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j_1)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j_1)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(98304 - 32768 * th >> 2) + (128 * j_1 + d_lane)] = dq_o; + pool_f32[(98304 - 32768 * th >> 2) + (1024 + 128 * j_1 + d_lane)] = dk_o; + } + float tot_0 = s_run_0[0]; + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1059[1]; - s_run_1059[0] = 0.0f; - int t_0 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew_1052 + 896 + d_lane]; - float dkf_0 = pool_f32[ew_1052 + 1920 + d_lane]; - float dgf_0 = pool_f32[ew_1052 + 2944 + d_lane]; - float bt_t_0 = btv_1058[7]; - float e_off_0 = approx_exp2(g_t_0 - gn_1049); - float e_dg_0 = approx_exp2(g_t_0 - gc_1050); - float dq2_0 = __fmaf_rn(_tmem_load_12[7], e_off_0, _tmem_load_14[7] * e_dg_0); - float dk2_0 = __fmaf_rn(_tmem_load_13[7], e_off_0, _tmem_load_15[7] * e_dg_0); - dbp_1057[7] = dk2_0 * k_t_0; - float dk2b_0 = dk2_0 * bt_t_0; - float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; - s_run_1059[0] = s_run_1059[0] + dg_o_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_0; - pool_f32[(98304 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_0; - int t_1 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew_1052 + 768 + d_lane]; - float dkf_1 = pool_f32[ew_1052 + 1792 + d_lane]; - float dgf_1 = pool_f32[ew_1052 + 2816 + d_lane]; - float bt_t_1 = btv_1058[6]; - float e_off_1 = approx_exp2(g_t_1 - gn_1049); - float e_dg_1 = approx_exp2(g_t_1 - gc_1050); - float dq2_1 = __fmaf_rn(_tmem_load_12[6], e_off_1, _tmem_load_14[6] * e_dg_1); - float dk2_1 = __fmaf_rn(_tmem_load_13[6], e_off_1, _tmem_load_15[6] * e_dg_1); - dbp_1057[6] = dk2_1 * k_t_1; - float dk2b_1 = dk2_1 * bt_t_1; - float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + _tmem_load_16[6]; - float dg_o_1 = __fmaf_rn(q_t_1_9, dq2_1, (dk2b_1 - _tmem_load_16[6]) * k_t_1) + dgf_1; - s_run_1059[0] = s_run_1059[0] + dg_o_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_1; - pool_f32[(98304 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_1; - int t_2 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew_1052 + 640 + d_lane]; - float dkf_2 = pool_f32[ew_1052 + 1664 + d_lane]; - float dgf_2 = pool_f32[ew_1052 + 2688 + d_lane]; - float bt_t_2 = btv_1058[5]; - float e_off_2 = approx_exp2(g_t_2 - gn_1049); - float e_dg_2 = approx_exp2(g_t_2 - gc_1050); - float dq2_2 = __fmaf_rn(_tmem_load_12[5], e_off_2, _tmem_load_14[5] * e_dg_2); - float dk2_2 = __fmaf_rn(_tmem_load_13[5], e_off_2, _tmem_load_15[5] * e_dg_2); - dbp_1057[5] = dk2_2 * k_t_2; - float dk2b_2 = dk2_2 * bt_t_2; - float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[5]; - float dg_o_2 = __fmaf_rn(q_t_1_10, dq2_2, (dk2b_2 - _tmem_load_16[5]) * k_t_2) + dgf_2; - s_run_1059[0] = s_run_1059[0] + dg_o_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_2; - pool_f32[(98304 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_2; - int t_3 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew_1052 + 512 + d_lane]; - float dkf_3 = pool_f32[ew_1052 + 1536 + d_lane]; - float dgf_3 = pool_f32[ew_1052 + 2560 + d_lane]; - float bt_t_3 = btv_1058[4]; - float e_off_3 = approx_exp2(g_t_3 - gn_1049); - float e_dg_3 = approx_exp2(g_t_3 - gc_1050); - float dq2_3 = __fmaf_rn(_tmem_load_12[4], e_off_3, _tmem_load_14[4] * e_dg_3); - float dk2_3 = __fmaf_rn(_tmem_load_13[4], e_off_3, _tmem_load_15[4] * e_dg_3); - dbp_1057[4] = dk2_3 * k_t_3; - float dk2b_3 = dk2_3 * bt_t_3; - float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[4]; - float dg_o_3 = __fmaf_rn(q_t_1_11, dq2_3, (dk2b_3 - _tmem_load_16[4]) * k_t_3) + dgf_3; - s_run_1059[0] = s_run_1059[0] + dg_o_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_3; - pool_f32[(98304 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_3; - int t_4 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew_1052 + 384 + d_lane]; - float dkf_4 = pool_f32[ew_1052 + 1408 + d_lane]; - float dgf_4 = pool_f32[ew_1052 + 2432 + d_lane]; - float bt_t_4 = btv_1058[3]; - float e_off_4 = approx_exp2(g_t_4 - gn_1049); - float e_dg_4 = approx_exp2(g_t_4 - gc_1050); - float dq2_4 = __fmaf_rn(_tmem_load_12[3], e_off_4, _tmem_load_14[3] * e_dg_4); - float dk2_4 = __fmaf_rn(_tmem_load_13[3], e_off_4, _tmem_load_15[3] * e_dg_4); - dbp_1057[3] = dk2_4 * k_t_4; - float dk2b_4 = dk2_4 * bt_t_4; - float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[3]; - float dg_o_4 = __fmaf_rn(q_t_1_12, dq2_4, (dk2b_4 - _tmem_load_16[3]) * k_t_4) + dgf_4; - s_run_1059[0] = s_run_1059[0] + dg_o_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_4; - pool_f32[(98304 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_4; - int t_5 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew_1052 + 256 + d_lane]; - float dkf_5 = pool_f32[ew_1052 + 1280 + d_lane]; - float dgf_5 = pool_f32[ew_1052 + 2304 + d_lane]; - float bt_t_5 = btv_1058[2]; - float e_off_5 = approx_exp2(g_t_5 - gn_1049); - float e_dg_5 = approx_exp2(g_t_5 - gc_1050); - float dq2_5 = __fmaf_rn(_tmem_load_12[2], e_off_5, _tmem_load_14[2] * e_dg_5); - float dk2_5 = __fmaf_rn(_tmem_load_13[2], e_off_5, _tmem_load_15[2] * e_dg_5); - dbp_1057[2] = dk2_5 * k_t_5; - float dk2b_5 = dk2_5 * bt_t_5; - float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[2]; - float dg_o_5 = __fmaf_rn(q_t_1_13, dq2_5, (dk2b_5 - _tmem_load_16[2]) * k_t_5) + dgf_5; - s_run_1059[0] = s_run_1059[0] + dg_o_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_5; - pool_f32[(98304 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_5; - int t_6 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew_1052 + 128 + d_lane]; - float dkf_6 = pool_f32[ew_1052 + 1152 + d_lane]; - float dgf_6 = pool_f32[ew_1052 + 2176 + d_lane]; - float bt_t_6 = btv_1058[1]; - float e_off_6 = approx_exp2(g_t_6 - gn_1049); - float e_dg_6 = approx_exp2(g_t_6 - gc_1050); - float dq2_6 = __fmaf_rn(_tmem_load_12[1], e_off_6, _tmem_load_14[1] * e_dg_6); - float dk2_6 = __fmaf_rn(_tmem_load_13[1], e_off_6, _tmem_load_15[1] * e_dg_6); - dbp_1057[1] = dk2_6 * k_t_6; - float dk2b_6 = dk2_6 * bt_t_6; - float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[1]; - float dg_o_6 = __fmaf_rn(q_t_1_14, dq2_6, (dk2b_6 - _tmem_load_16[1]) * k_t_6) + dgf_6; - s_run_1059[0] = s_run_1059[0] + dg_o_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_6; - pool_f32[(98304 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_6; - int t_7 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew_1052 + d_lane]; - float dkf_7 = pool_f32[ew_1052 + 1024 + d_lane]; - float dgf_7 = pool_f32[ew_1052 + 2048 + d_lane]; - float bt_t_7 = btv_1058[0]; - float e_off_7 = approx_exp2(g_t_7 - gn_1049); - float e_dg_7 = approx_exp2(g_t_7 - gc_1050); - float dq2_7 = __fmaf_rn(_tmem_load_12[0], e_off_7, _tmem_load_14[0] * e_dg_7); - float dk2_7 = __fmaf_rn(_tmem_load_13[0], e_off_7, _tmem_load_15[0] * e_dg_7); - dbp_1057[0] = dk2_7 * k_t_7; - float dk2b_7 = dk2_7 * bt_t_7; - float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[0]; - float dg_o_7 = __fmaf_rn(q_t_1_15, dq2_7, (dk2b_7 - _tmem_load_16[0]) * k_t_7) + dgf_7; - s_run_1059[0] = s_run_1059[0] + dg_o_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + d_lane] = dq_o_7; - pool_f32[(98304 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_7; - float tot_1228 = s_run_1059[0]; - int b4_1229 = lane_1 >> 4 & 1; - int b3_1230 = lane_1 >> 3 & 1; - int b2_1231 = lane_1 >> 2 & 1; - float v4_1232[4]; - float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_316 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_316; - float send_1 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_317 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_317; - float send_2 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_318 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_318; - float send_3 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_319 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_319; - float v2_1237[2]; - float send_4 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_320 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_320; - float send_5 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_321 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_321; - float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_322 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_322; - float _shfl_xor_sync_323 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_323; - float _shfl_xor_sync_324 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_324; + int b4_0 = lane_1 >> 4 & 1; + int b3_0 = lane_1 >> 3 & 1; + int b2_0 = lane_1 >> 2 & 1; + float v4_0[4]; + float send_0 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); + float _shfl_xor_sync_348 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_348; + float send_1 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); + float _shfl_xor_sync_349 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_349; + float send_2 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); + float _shfl_xor_sync_350 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_350; + float send_3 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); + float _shfl_xor_sync_351 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_351; + float v2[2]; + float send2_0 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); + float _shfl_xor_sync_352 = __shfl_xor_sync(0xFFFFFFFF, send2_0, 8); + v2[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_352; + float send2_1 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); + float _shfl_xor_sync_353 = __shfl_xor_sync(0xFFFFFFFF, send2_1, 8); + v2[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_353; + float send1 = ((b2_0 == 0) ? v2[1] : v2[0]); + float _shfl_xor_sync_354 = __shfl_xor_sync(0xFFFFFFFF, send1, 4); + float v1_0 = ((b2_0 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_354; + float _shfl_xor_sync_355 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_355; + float _shfl_xor_sync_356 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_356; asm volatile("tcgen05.fence::before_thread_sync;"); - int ew_1 = 98304 - 32768 * th >> 2; + int ew_2 = 98304 - 32768 * th >> 2; float nz_0 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); asm volatile("barrier.sync 1, 256;" ::: "memory"); #pragma unroll 1 for (int i_1 = 0; i_1 < 2; i_1++) { - int j_1 = qd + 4 * i_1; - bool live_1 = 48 + 8 * th + j_1 < clen; - int _min_325 = ((48 + 8 * th + j_1) < (clen - 1) ? (48 + 8 * th + j_1) : (clen - 1)); - long long row_1 = ext0 + (long long)_min_325; + int j_2 = qd + 4 * i_1; + bool live_1 = 48 + 8 * th + j_2 < clen; + int _min_357 = ((48 + 8 * th + j_2) < (clen - 1) ? (48 + 8 * th + j_2) : (clen - 1)); + long long row_1 = ext0 + (long long)_min_357; float gq_1[4]; float gk_1[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gq_1[0])), "=r"(*reinterpret_cast(&gq_1[(0) + 1])), "=r"(*reinterpret_cast(&gq_1[(0) + 2])), "=r"(*reinterpret_cast(&gq_1[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_1 + 128 * j_1 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_2 + 128 * j_2 + 4 * lane_1)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gk_1[0])), "=r"(*reinterpret_cast(&gk_1[(0) + 1])), "=r"(*reinterpret_cast(&gk_1[(0) + 2])), "=r"(*reinterpret_cast(&gk_1[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_1 + 1024 + 128 * j_1 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_2 + 1024 + 128 * j_2 + 4 * lane_1)))); if (i_1 == 1) { mbarrier_arrive(epi_empty_addr + (th) * 8); } @@ -3887,26 +3852,26 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float rq_1[1]; float rk_1[1]; { - uint2 _vld_326; - _vld_326 = *reinterpret_cast(q_raw + (row_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_326 = reinterpret_cast(&_vld_326); + uint2 _vld_358; + _vld_358 = *reinterpret_cast(q_raw + (row_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_358 = reinterpret_cast(&_vld_358); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xq_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_326[_pair]) << 16); - (&xq_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_326[_pair]) & 0xffff0000u); + (&xq_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_358[_pair]) << 16); + (&xq_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_358[_pair]) & 0xffff0000u); } } { rq_1[0] = *reinterpret_cast(qk_rstd + ((0 + row_1) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_327; - _vld_327 = *reinterpret_cast(k_raw + (row_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_327 = reinterpret_cast(&_vld_327); + uint2 _vld_359; + _vld_359 = *reinterpret_cast(k_raw + (row_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_359 = reinterpret_cast(&_vld_359); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xk_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_327[_pair]) << 16); - (&xk_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_327[_pair]) & 0xffff0000u); + (&xk_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_359[_pair]) << 16); + (&xk_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_359[_pair]) & 0xffff0000u); } } { @@ -3916,106 +3881,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_1[4]; float p_1[4]; y_1[0] = mul_f32_rn_noftz(xq_1[0], rq_1[0]); - float _fma_328; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_328) : "f"(gq_1[0]), "f"(y_1[0]), "f"(nz_0)); - p_1[0] = _fma_328; + float _fma_360; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_360) : "f"(gq_1[0]), "f"(y_1[0]), "f"(nz_0)); + p_1[0] = _fma_360; y_1[1] = mul_f32_rn_noftz(xq_1[1], rq_1[0]); - float _fma_329; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_329) : "f"(gq_1[1]), "f"(y_1[1]), "f"(nz_0)); - p_1[1] = _fma_329; + float _fma_361; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_361) : "f"(gq_1[1]), "f"(y_1[1]), "f"(nz_0)); + p_1[1] = _fma_361; y_1[2] = mul_f32_rn_noftz(xq_1[2], rq_1[0]); - float _fma_330; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_330) : "f"(gq_1[2]), "f"(y_1[2]), "f"(nz_0)); - p_1[2] = _fma_330; + float _fma_362; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_362) : "f"(gq_1[2]), "f"(y_1[2]), "f"(nz_0)); + p_1[2] = _fma_362; y_1[3] = mul_f32_rn_noftz(xq_1[3], rq_1[0]); - float _fma_331; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_331) : "f"(gq_1[3]), "f"(y_1[3]), "f"(nz_0)); - p_1[3] = _fma_331; - float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 4); - float _add_332; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_332) : "f"(p_1[0]), "f"(_shfl_xor_sync_333)); - p_1[0] = _add_332; - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 4); - float _add_334; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(p_1[1]), "f"(_shfl_xor_sync_335)); - p_1[1] = _add_334; - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 4); - float _add_336; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_336) : "f"(p_1[2]), "f"(_shfl_xor_sync_337)); - p_1[2] = _add_336; - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 4); - float _add_338; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_338) : "f"(p_1[3]), "f"(_shfl_xor_sync_339)); - p_1[3] = _add_338; - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 2); - float _add_340; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_340) : "f"(p_1[0]), "f"(_shfl_xor_sync_341)); - p_1[0] = _add_340; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 2); - float _add_342; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_342) : "f"(p_1[1]), "f"(_shfl_xor_sync_343)); - p_1[1] = _add_342; - float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 2); - float _add_344; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_344) : "f"(p_1[2]), "f"(_shfl_xor_sync_345)); - p_1[2] = _add_344; - float _shfl_xor_sync_347 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 2); - float _add_346; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_346) : "f"(p_1[3]), "f"(_shfl_xor_sync_347)); - p_1[3] = _add_346; - float _shfl_xor_sync_349 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 1); - float _add_348; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_348) : "f"(p_1[0]), "f"(_shfl_xor_sync_349)); - p_1[0] = _add_348; - float _shfl_xor_sync_351 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 1); - float _add_350; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_350) : "f"(p_1[1]), "f"(_shfl_xor_sync_351)); - p_1[1] = _add_350; - float _shfl_xor_sync_353 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 1); - float _add_352; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_352) : "f"(p_1[2]), "f"(_shfl_xor_sync_353)); - p_1[2] = _add_352; - float _shfl_xor_sync_355 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 1); - float _add_354; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_354) : "f"(p_1[3]), "f"(_shfl_xor_sync_355)); - p_1[3] = _add_354; - float _add_356; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_356) : "f"(p_1[0]), "f"(p_1[2])); - float _add_357; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_357) : "f"(p_1[1]), "f"(p_1[3])); + float _fma_363; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_363) : "f"(gq_1[3]), "f"(y_1[3]), "f"(nz_0)); + p_1[3] = _fma_363; + float _shfl_xor_sync_365 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 4); + float _add_364; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_364) : "f"(p_1[0]), "f"(_shfl_xor_sync_365)); + p_1[0] = _add_364; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 4); + float _add_366; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_366) : "f"(p_1[1]), "f"(_shfl_xor_sync_367)); + p_1[1] = _add_366; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 4); + float _add_368; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_368) : "f"(p_1[2]), "f"(_shfl_xor_sync_369)); + p_1[2] = _add_368; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 4); + float _add_370; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_370) : "f"(p_1[3]), "f"(_shfl_xor_sync_371)); + p_1[3] = _add_370; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 2); + float _add_372; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_372) : "f"(p_1[0]), "f"(_shfl_xor_sync_373)); + p_1[0] = _add_372; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 2); + float _add_374; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_374) : "f"(p_1[1]), "f"(_shfl_xor_sync_375)); + p_1[1] = _add_374; + float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 2); + float _add_376; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_376) : "f"(p_1[2]), "f"(_shfl_xor_sync_377)); + p_1[2] = _add_376; + float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 2); + float _add_378; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_378) : "f"(p_1[3]), "f"(_shfl_xor_sync_379)); + p_1[3] = _add_378; + float _shfl_xor_sync_381 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 1); + float _add_380; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_380) : "f"(p_1[0]), "f"(_shfl_xor_sync_381)); + p_1[0] = _add_380; + float _shfl_xor_sync_383 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 1); + float _add_382; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_382) : "f"(p_1[1]), "f"(_shfl_xor_sync_383)); + p_1[1] = _add_382; + float _shfl_xor_sync_385 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 1); + float _add_384; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_384) : "f"(p_1[2]), "f"(_shfl_xor_sync_385)); + p_1[2] = _add_384; + float _shfl_xor_sync_387 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 1); + float _add_386; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_386) : "f"(p_1[3]), "f"(_shfl_xor_sync_387)); + p_1[3] = _add_386; + float _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(p_1[0]), "f"(p_1[2])); + float _add_389; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_389) : "f"(p_1[1]), "f"(p_1[3])); float sq_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1) : "f"(_add_356), "f"(_add_357)); - float _shfl_sync_360 = __shfl_sync(0xFFFFFFFF, sq_1, 0); - float _shfl_sync_361 = __shfl_sync(0xFFFFFFFF, sq_1, 8); - float _add_359; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_359) : "f"(_shfl_sync_360), "f"(_shfl_sync_361)); - float _shfl_sync_362 = __shfl_sync(0xFFFFFFFF, sq_1, 16); - float _add_358; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_358) : "f"(_add_359), "f"(_shfl_sync_362)); - float _shfl_sync_363 = __shfl_sync(0xFFFFFFFF, sq_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1) : "f"(_add_388), "f"(_add_389)); + float _shfl_sync_392 = __shfl_sync(0xFFFFFFFF, sq_1, 0); + float _shfl_sync_393 = __shfl_sync(0xFFFFFFFF, sq_1, 8); + float _add_391; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_391) : "f"(_shfl_sync_392), "f"(_shfl_sync_393)); + float _shfl_sync_394 = __shfl_sync(0xFFFFFFFF, sq_1, 16); + float _add_390; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_390) : "f"(_add_391), "f"(_shfl_sync_394)); + float _shfl_sync_395 = __shfl_sync(0xFFFFFFFF, sq_1, 24); float dot_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_358), "f"(_shfl_sync_363)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_390), "f"(_shfl_sync_395)); float o_1[4]; - float _fma_365; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_365) : "f"(y_1[0]), "f"(dot_1), "f"(nz_0)); - float _sub_364; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_364) : "f"(gq_1[0]), "f"(_fma_365)); - o_1[0] = mul_f32_rn_noftz(rq_1[0], _sub_364); - float _fma_367; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_367) : "f"(y_1[1]), "f"(dot_1), "f"(nz_0)); - float _sub_366; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_366) : "f"(gq_1[1]), "f"(_fma_367)); - o_1[1] = mul_f32_rn_noftz(rq_1[0], _sub_366); - float _fma_369; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_369) : "f"(y_1[2]), "f"(dot_1), "f"(nz_0)); - float _sub_368; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_368) : "f"(gq_1[2]), "f"(_fma_369)); - o_1[2] = mul_f32_rn_noftz(rq_1[0], _sub_368); - float _fma_371; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_371) : "f"(y_1[3]), "f"(dot_1), "f"(nz_0)); - float _sub_370; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_370) : "f"(gq_1[3]), "f"(_fma_371)); - o_1[3] = mul_f32_rn_noftz(rq_1[0], _sub_370); + float _fma_397; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_397) : "f"(y_1[0]), "f"(dot_1), "f"(nz_0)); + float _sub_396; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_396) : "f"(gq_1[0]), "f"(_fma_397)); + o_1[0] = mul_f32_rn_noftz(rq_1[0], _sub_396); + float _fma_399; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_399) : "f"(y_1[1]), "f"(dot_1), "f"(nz_0)); + float _sub_398; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_398) : "f"(gq_1[1]), "f"(_fma_399)); + o_1[1] = mul_f32_rn_noftz(rq_1[0], _sub_398); + float _fma_401; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_401) : "f"(y_1[2]), "f"(dot_1), "f"(nz_0)); + float _sub_400; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_400) : "f"(gq_1[2]), "f"(_fma_401)); + o_1[2] = mul_f32_rn_noftz(rq_1[0], _sub_400); + float _fma_403; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_403) : "f"(y_1[3]), "f"(dot_1), "f"(nz_0)); + float _sub_402; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_402) : "f"(gq_1[3]), "f"(_fma_403)); + o_1[3] = mul_f32_rn_noftz(rq_1[0], _sub_402); unsigned int ob_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -4026,106 +3991,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_1_1[4]; float p_1_1[4]; y_1_1[0] = mul_f32_rn_noftz(xk_1[0], rk_1[0]); - float _fma_372; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_372) : "f"(gk_1[0]), "f"(y_1_1[0]), "f"(nz_0)); - p_1_1[0] = _fma_372; + float _fma_404; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_404) : "f"(gk_1[0]), "f"(y_1_1[0]), "f"(nz_0)); + p_1_1[0] = _fma_404; y_1_1[1] = mul_f32_rn_noftz(xk_1[1], rk_1[0]); - float _fma_373; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_373) : "f"(gk_1[1]), "f"(y_1_1[1]), "f"(nz_0)); - p_1_1[1] = _fma_373; + float _fma_405; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_405) : "f"(gk_1[1]), "f"(y_1_1[1]), "f"(nz_0)); + p_1_1[1] = _fma_405; y_1_1[2] = mul_f32_rn_noftz(xk_1[2], rk_1[0]); - float _fma_374; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_374) : "f"(gk_1[2]), "f"(y_1_1[2]), "f"(nz_0)); - p_1_1[2] = _fma_374; + float _fma_406; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_406) : "f"(gk_1[2]), "f"(y_1_1[2]), "f"(nz_0)); + p_1_1[2] = _fma_406; y_1_1[3] = mul_f32_rn_noftz(xk_1[3], rk_1[0]); - float _fma_375; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_375) : "f"(gk_1[3]), "f"(y_1_1[3]), "f"(nz_0)); - p_1_1[3] = _fma_375; - float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 4); - float _add_376; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_376) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_377)); - p_1_1[0] = _add_376; - float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 4); - float _add_378; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_378) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_379)); - p_1_1[1] = _add_378; - float _shfl_xor_sync_381 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 4); - float _add_380; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_380) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_381)); - p_1_1[2] = _add_380; - float _shfl_xor_sync_383 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 4); - float _add_382; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_382) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_383)); - p_1_1[3] = _add_382; - float _shfl_xor_sync_385 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 2); - float _add_384; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_384) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_385)); - p_1_1[0] = _add_384; - float _shfl_xor_sync_387 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 2); - float _add_386; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_386) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_387)); - p_1_1[1] = _add_386; - float _shfl_xor_sync_389 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 2); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_389)); - p_1_1[2] = _add_388; - float _shfl_xor_sync_391 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 2); - float _add_390; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_390) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_391)); - p_1_1[3] = _add_390; - float _shfl_xor_sync_393 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 1); - float _add_392; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_393)); - p_1_1[0] = _add_392; - float _shfl_xor_sync_395 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 1); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_395)); - p_1_1[1] = _add_394; - float _shfl_xor_sync_397 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 1); - float _add_396; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_396) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_397)); - p_1_1[2] = _add_396; - float _shfl_xor_sync_399 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 1); - float _add_398; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_399)); - p_1_1[3] = _add_398; - float _add_400; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_400) : "f"(p_1_1[0]), "f"(p_1_1[2])); - float _add_401; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_401) : "f"(p_1_1[1]), "f"(p_1_1[3])); + float _fma_407; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_407) : "f"(gk_1[3]), "f"(y_1_1[3]), "f"(nz_0)); + p_1_1[3] = _fma_407; + float _shfl_xor_sync_409 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 4); + float _add_408; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_408) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_409)); + p_1_1[0] = _add_408; + float _shfl_xor_sync_411 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 4); + float _add_410; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_410) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_411)); + p_1_1[1] = _add_410; + float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 4); + float _add_412; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_412) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_413)); + p_1_1[2] = _add_412; + float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 4); + float _add_414; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_414) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_415)); + p_1_1[3] = _add_414; + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 2); + float _add_416; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_416) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_417)); + p_1_1[0] = _add_416; + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 2); + float _add_418; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_418) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_419)); + p_1_1[1] = _add_418; + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 2); + float _add_420; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_420) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_421)); + p_1_1[2] = _add_420; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 2); + float _add_422; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_422) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_423)); + p_1_1[3] = _add_422; + float _shfl_xor_sync_425 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 1); + float _add_424; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_424) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_425)); + p_1_1[0] = _add_424; + float _shfl_xor_sync_427 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 1); + float _add_426; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_426) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_427)); + p_1_1[1] = _add_426; + float _shfl_xor_sync_429 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 1); + float _add_428; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_428) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_429)); + p_1_1[2] = _add_428; + float _shfl_xor_sync_431 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 1); + float _add_430; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_430) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_431)); + p_1_1[3] = _add_430; + float _add_432; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_432) : "f"(p_1_1[0]), "f"(p_1_1[2])); + float _add_433; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_433) : "f"(p_1_1[1]), "f"(p_1_1[3])); float sq_1_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1_1) : "f"(_add_400), "f"(_add_401)); - float _shfl_sync_404 = __shfl_sync(0xFFFFFFFF, sq_1_1, 0); - float _shfl_sync_405 = __shfl_sync(0xFFFFFFFF, sq_1_1, 8); - float _add_403; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_403) : "f"(_shfl_sync_404), "f"(_shfl_sync_405)); - float _shfl_sync_406 = __shfl_sync(0xFFFFFFFF, sq_1_1, 16); - float _add_402; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_402) : "f"(_add_403), "f"(_shfl_sync_406)); - float _shfl_sync_407 = __shfl_sync(0xFFFFFFFF, sq_1_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1_1) : "f"(_add_432), "f"(_add_433)); + float _shfl_sync_436 = __shfl_sync(0xFFFFFFFF, sq_1_1, 0); + float _shfl_sync_437 = __shfl_sync(0xFFFFFFFF, sq_1_1, 8); + float _add_435; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_435) : "f"(_shfl_sync_436), "f"(_shfl_sync_437)); + float _shfl_sync_438 = __shfl_sync(0xFFFFFFFF, sq_1_1, 16); + float _add_434; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_434) : "f"(_add_435), "f"(_shfl_sync_438)); + float _shfl_sync_439 = __shfl_sync(0xFFFFFFFF, sq_1_1, 24); float dot_1_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1_1) : "f"(_add_402), "f"(_shfl_sync_407)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1_1) : "f"(_add_434), "f"(_shfl_sync_439)); float o_1_1[4]; - float _fma_409; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_409) : "f"(y_1_1[0]), "f"(dot_1_1), "f"(nz_0)); - float _sub_408; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_408) : "f"(gk_1[0]), "f"(_fma_409)); - o_1_1[0] = mul_f32_rn_noftz(rk_1[0], _sub_408); - float _fma_411; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_411) : "f"(y_1_1[1]), "f"(dot_1_1), "f"(nz_0)); - float _sub_410; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_410) : "f"(gk_1[1]), "f"(_fma_411)); - o_1_1[1] = mul_f32_rn_noftz(rk_1[0], _sub_410); - float _fma_413; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_413) : "f"(y_1_1[2]), "f"(dot_1_1), "f"(nz_0)); - float _sub_412; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_412) : "f"(gk_1[2]), "f"(_fma_413)); - o_1_1[2] = mul_f32_rn_noftz(rk_1[0], _sub_412); - float _fma_415; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_415) : "f"(y_1_1[3]), "f"(dot_1_1), "f"(nz_0)); - float _sub_414; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_414) : "f"(gk_1[3]), "f"(_fma_415)); - o_1_1[3] = mul_f32_rn_noftz(rk_1[0], _sub_414); + float _fma_441; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_441) : "f"(y_1_1[0]), "f"(dot_1_1), "f"(nz_0)); + float _sub_440; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_440) : "f"(gk_1[0]), "f"(_fma_441)); + o_1_1[0] = mul_f32_rn_noftz(rk_1[0], _sub_440); + float _fma_443; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_443) : "f"(y_1_1[1]), "f"(dot_1_1), "f"(nz_0)); + float _sub_442; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_442) : "f"(gk_1[1]), "f"(_fma_443)); + o_1_1[1] = mul_f32_rn_noftz(rk_1[0], _sub_442); + float _fma_445; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_445) : "f"(y_1_1[2]), "f"(dot_1_1), "f"(nz_0)); + float _sub_444; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_444) : "f"(gk_1[2]), "f"(_fma_445)); + o_1_1[2] = mul_f32_rn_noftz(rk_1[0], _sub_444); + float _fma_447; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_447) : "f"(y_1_1[3]), "f"(dot_1_1), "f"(nz_0)); + float _sub_446; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_446) : "f"(gk_1[3]), "f"(_fma_447)); + o_1_1[3] = mul_f32_rn_noftz(rk_1[0], _sub_446); unsigned int ob_1_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -4183,8 +4148,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; { - uint32_t _addr_416 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_416), "f"(tot_1228) : "memory"); + uint32_t _addr_448 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_448), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4202,8 +4167,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi = ((th == 1) ? tot_1228 : t_other[0]); - float t_lo = ((th == 1) ? t_other[0] : tot_1228); + float t_hi = ((th == 1) ? tot_0 : t_other[0]); + float t_lo = ((th == 1) ? t_other[0] : tot_0); float offset = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi); float sfx[8]; tmem_ld_x8(&sfx[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); @@ -4216,132 +4181,132 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_417 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_449 = make_float2(gacc[4], gacc[4]); float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_417), "l"(*(const unsigned long long*)&rp_u)); - float _abs_420; - asm("abs.f32 %0, %1;" : "=f"(_abs_420) : "f"(rp_z.x)); - float _neg_419; - asm("neg.f32 %0, %1;" : "=f"(_neg_419) : "f"(_abs_420)); - float _exp_418 = __nv_expf(_neg_419); - float _abs_423; - asm("abs.f32 %0, %1;" : "=f"(_abs_423) : "f"(rp_z.y)); - float _neg_422; - asm("neg.f32 %0, %1;" : "=f"(_neg_422) : "f"(_abs_423)); - float _exp_421 = __nv_expf(_neg_422); - float2 rp_e = make_float2(_exp_418, _exp_421); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_449), "l"(*(const unsigned long long*)&rp_u)); + float _abs_452; + asm("abs.f32 %0, %1;" : "=f"(_abs_452) : "f"(rp_z.x)); + float _neg_451; + asm("neg.f32 %0, %1;" : "=f"(_neg_451) : "f"(_abs_452)); + float _exp_450 = __nv_expf(_neg_451); + float _abs_455; + asm("abs.f32 %0, %1;" : "=f"(_abs_455) : "f"(rp_z.y)); + float _neg_454; + asm("neg.f32 %0, %1;" : "=f"(_neg_454) : "f"(_abs_455)); + float _exp_453 = __nv_expf(_neg_454); + float2 rp_e = make_float2(_exp_450, _exp_453); float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_424 = approx_rcp(rp_den.x); - float _rcp_425 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_426 = make_float2(_rcp_424, _rcp_425); + float _rcp_456 = approx_rcp(rp_den.x); + float _rcp_457 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_458 = make_float2(_rcp_456, _rcp_457); float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_426)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_427 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_459 = make_float2(gacc[5], gacc[5]); float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_427)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_459)); u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_428 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_460 = make_float2(gacc[4], gacc[4]); float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_428), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_431; - asm("abs.f32 %0, %1;" : "=f"(_abs_431) : "f"(rp_z_0.x)); - float _neg_430; - asm("neg.f32 %0, %1;" : "=f"(_neg_430) : "f"(_abs_431)); - float _exp_429 = __nv_expf(_neg_430); - float _abs_434; - asm("abs.f32 %0, %1;" : "=f"(_abs_434) : "f"(rp_z_0.y)); - float _neg_433; - asm("neg.f32 %0, %1;" : "=f"(_neg_433) : "f"(_abs_434)); - float _exp_432 = __nv_expf(_neg_433); - float2 rp_e_0 = make_float2(_exp_429, _exp_432); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_460), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_463; + asm("abs.f32 %0, %1;" : "=f"(_abs_463) : "f"(rp_z_0.x)); + float _neg_462; + asm("neg.f32 %0, %1;" : "=f"(_neg_462) : "f"(_abs_463)); + float _exp_461 = __nv_expf(_neg_462); + float _abs_466; + asm("abs.f32 %0, %1;" : "=f"(_abs_466) : "f"(rp_z_0.y)); + float _neg_465; + asm("neg.f32 %0, %1;" : "=f"(_neg_465) : "f"(_abs_466)); + float _exp_464 = __nv_expf(_neg_465); + float2 rp_e_0 = make_float2(_exp_461, _exp_464); float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_435 = approx_rcp(rp_den_0.x); - float _rcp_436 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_437 = make_float2(_rcp_435, _rcp_436); + float _rcp_467 = approx_rcp(rp_den_0.x); + float _rcp_468 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_469 = make_float2(_rcp_467, _rcp_468); float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_437)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_438 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_470 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_438)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_470)); u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_439 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_471 = make_float2(gacc[4], gacc[4]); float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_439), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_442; - asm("abs.f32 %0, %1;" : "=f"(_abs_442) : "f"(rp_z_1.x)); - float _neg_441; - asm("neg.f32 %0, %1;" : "=f"(_neg_441) : "f"(_abs_442)); - float _exp_440 = __nv_expf(_neg_441); - float _abs_445; - asm("abs.f32 %0, %1;" : "=f"(_abs_445) : "f"(rp_z_1.y)); - float _neg_444; - asm("neg.f32 %0, %1;" : "=f"(_neg_444) : "f"(_abs_445)); - float _exp_443 = __nv_expf(_neg_444); - float2 rp_e_1 = make_float2(_exp_440, _exp_443); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_471), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_474; + asm("abs.f32 %0, %1;" : "=f"(_abs_474) : "f"(rp_z_1.x)); + float _neg_473; + asm("neg.f32 %0, %1;" : "=f"(_neg_473) : "f"(_abs_474)); + float _exp_472 = __nv_expf(_neg_473); + float _abs_477; + asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(rp_z_1.y)); + float _neg_476; + asm("neg.f32 %0, %1;" : "=f"(_neg_476) : "f"(_abs_477)); + float _exp_475 = __nv_expf(_neg_476); + float2 rp_e_1 = make_float2(_exp_472, _exp_475); float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_446 = approx_rcp(rp_den_1.x); - float _rcp_447 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_448 = make_float2(_rcp_446, _rcp_447); + float _rcp_478 = approx_rcp(rp_den_1.x); + float _rcp_479 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_480 = make_float2(_rcp_478, _rcp_479); float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_448)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_480)); float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_449 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_481 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_449)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_481)); u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_450 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_482 = make_float2(gacc[4], gacc[4]); float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_450), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_453; - asm("abs.f32 %0, %1;" : "=f"(_abs_453) : "f"(rp_z_2.x)); - float _neg_452; - asm("neg.f32 %0, %1;" : "=f"(_neg_452) : "f"(_abs_453)); - float _exp_451 = __nv_expf(_neg_452); - float _abs_456; - asm("abs.f32 %0, %1;" : "=f"(_abs_456) : "f"(rp_z_2.y)); - float _neg_455; - asm("neg.f32 %0, %1;" : "=f"(_neg_455) : "f"(_abs_456)); - float _exp_454 = __nv_expf(_neg_455); - float2 rp_e_2 = make_float2(_exp_451, _exp_454); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_482), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_485; + asm("abs.f32 %0, %1;" : "=f"(_abs_485) : "f"(rp_z_2.x)); + float _neg_484; + asm("neg.f32 %0, %1;" : "=f"(_neg_484) : "f"(_abs_485)); + float _exp_483 = __nv_expf(_neg_484); + float _abs_488; + asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(rp_z_2.y)); + float _neg_487; + asm("neg.f32 %0, %1;" : "=f"(_neg_487) : "f"(_abs_488)); + float _exp_486 = __nv_expf(_neg_487); + float2 rp_e_2 = make_float2(_exp_483, _exp_486); float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_457 = approx_rcp(rp_den_2.x); - float _rcp_458 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_459 = make_float2(_rcp_457, _rcp_458); + float _rcp_489 = approx_rcp(rp_den_2.x); + float _rcp_490 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_491 = make_float2(_rcp_489, _rcp_490); float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_459)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_491)); float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_460 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_492 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_460)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_492)); u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -4356,54 +4321,54 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_461; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_461) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_461; + uint32_t _prmt_493; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_493) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_493; int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_462 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_462)); - *(reinterpret_cast(_gst_462) + (0)) = pq_0[0]; + auto _gst_494 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_494)); + *(reinterpret_cast(_gst_494) + (0)) = pq_0[0]; } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_463; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_463) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_463; + uint32_t _prmt_495; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_495) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_495; int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_464 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_464)); - *(reinterpret_cast(_gst_464) + (0)) = pq_1[0]; + auto _gst_496 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_496)); + *(reinterpret_cast(_gst_496) + (0)) = pq_1[0]; } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_465; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_465) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_465; + uint32_t _prmt_497; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_497) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_497; int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_466 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_466)); - *(reinterpret_cast(_gst_466) + (0)) = pq_2[0]; + auto _gst_498 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_498)); + *(reinterpret_cast(_gst_498) + (0)) = pq_2[0]; } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_467; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_467) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_467; + uint32_t _prmt_499; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_499) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_499; int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_468 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_468)); - *(reinterpret_cast(_gst_468) + (0)) = pq_3[0]; + auto _gst_500 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_500)); + *(reinterpret_cast(_gst_500) + (0)) = pq_3[0]; } } int t_a_0 = 48 + 8 * th; @@ -4468,47 +4433,47 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_469; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_469) : "f"(s_a), "f"(nm_t)); + float _sub_501; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_501) : "f"(s_a), "f"(nm_t)); float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_469), "f"(pa)); - float _sub_470; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_470) : "f"(pa), "f"(nm_t)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_501), "f"(pa)); + float _sub_502; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_502) : "f"(pa), "f"(nm_t)); float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_470), "f"(s_a)); - float _abs_472; - asm("abs.f32 %0, %1;" : "=f"(_abs_472) : "f"(s_a)); - float _abs_473; - asm("abs.f32 %0, %1;" : "=f"(_abs_473) : "f"(pa)); - uint32_t _setp_471_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_471_bits) : "f"(_abs_472), "f"(_abs_473)); - bool _setp_471 = _setp_471_bits != 0; - float nm_err = ((_setp_471) ? nm_es : nm_ex); - float _add_474; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_474) : "f"(c_a), "f"(nm_err)); - c_a = _add_474; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_502), "f"(s_a)); + float _abs_504; + asm("abs.f32 %0, %1;" : "=f"(_abs_504) : "f"(s_a)); + float _abs_505; + asm("abs.f32 %0, %1;" : "=f"(_abs_505) : "f"(pa)); + uint32_t _setp_503_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_503_bits) : "f"(_abs_504), "f"(_abs_505)); + bool _setp_503 = _setp_503_bits != 0; + float nm_err = ((_setp_503) ? nm_es : nm_ex); + float _add_506; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_506) : "f"(c_a), "f"(nm_err)); + c_a = _add_506; s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_475; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_475) : "f"(s_b), "f"(nm_t_0)); + float _sub_507; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_507) : "f"(s_b), "f"(nm_t_0)); float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_475), "f"(pb)); - float _sub_476; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_476) : "f"(pb), "f"(nm_t_0)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_507), "f"(pb)); + float _sub_508; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_508) : "f"(pb), "f"(nm_t_0)); float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_476), "f"(s_b)); - float _abs_478; - asm("abs.f32 %0, %1;" : "=f"(_abs_478) : "f"(s_b)); - float _abs_479; - asm("abs.f32 %0, %1;" : "=f"(_abs_479) : "f"(pb)); - uint32_t _setp_477_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_477_bits) : "f"(_abs_478), "f"(_abs_479)); - bool _setp_477 = _setp_477_bits != 0; - float nm_err_0 = ((_setp_477) ? nm_es_0 : nm_ex_0); - float _add_480; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_480) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_480; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_508), "f"(s_b)); + float _abs_510; + asm("abs.f32 %0, %1;" : "=f"(_abs_510) : "f"(s_b)); + float _abs_511; + asm("abs.f32 %0, %1;" : "=f"(_abs_511) : "f"(pb)); + uint32_t _setp_509_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_509_bits) : "f"(_abs_510), "f"(_abs_511)); + bool _setp_509 = _setp_509_bits != 0; + float nm_err_0 = ((_setp_509) ? nm_es_0 : nm_ex_0); + float _add_512; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_512) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_512; s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4532,132 +4497,132 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_481 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_513 = make_float2(gacc[4], gacc[4]); float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_481), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_484; - asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(rp_z_3.x)); - float _neg_483; - asm("neg.f32 %0, %1;" : "=f"(_neg_483) : "f"(_abs_484)); - float _exp_482 = __nv_expf(_neg_483); - float _abs_487; - asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(rp_z_3.y)); - float _neg_486; - asm("neg.f32 %0, %1;" : "=f"(_neg_486) : "f"(_abs_487)); - float _exp_485 = __nv_expf(_neg_486); - float2 rp_e_3 = make_float2(_exp_482, _exp_485); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_513), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_516; + asm("abs.f32 %0, %1;" : "=f"(_abs_516) : "f"(rp_z_3.x)); + float _neg_515; + asm("neg.f32 %0, %1;" : "=f"(_neg_515) : "f"(_abs_516)); + float _exp_514 = __nv_expf(_neg_515); + float _abs_519; + asm("abs.f32 %0, %1;" : "=f"(_abs_519) : "f"(rp_z_3.y)); + float _neg_518; + asm("neg.f32 %0, %1;" : "=f"(_neg_518) : "f"(_abs_519)); + float _exp_517 = __nv_expf(_neg_518); + float2 rp_e_3 = make_float2(_exp_514, _exp_517); float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_488 = approx_rcp(rp_den_3.x); - float _rcp_489 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_490 = make_float2(_rcp_488, _rcp_489); + float _rcp_520 = approx_rcp(rp_den_3.x); + float _rcp_521 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_522 = make_float2(_rcp_520, _rcp_521); float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_490)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_522)); float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_491 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_523 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_491)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_492 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_524 = make_float2(gacc[4], gacc[4]); float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_492), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_495; - asm("abs.f32 %0, %1;" : "=f"(_abs_495) : "f"(rp_z_4.x)); - float _neg_494; - asm("neg.f32 %0, %1;" : "=f"(_neg_494) : "f"(_abs_495)); - float _exp_493 = __nv_expf(_neg_494); - float _abs_498; - asm("abs.f32 %0, %1;" : "=f"(_abs_498) : "f"(rp_z_4.y)); - float _neg_497; - asm("neg.f32 %0, %1;" : "=f"(_neg_497) : "f"(_abs_498)); - float _exp_496 = __nv_expf(_neg_497); - float2 rp_e_4 = make_float2(_exp_493, _exp_496); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_524), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_527; + asm("abs.f32 %0, %1;" : "=f"(_abs_527) : "f"(rp_z_4.x)); + float _neg_526; + asm("neg.f32 %0, %1;" : "=f"(_neg_526) : "f"(_abs_527)); + float _exp_525 = __nv_expf(_neg_526); + float _abs_530; + asm("abs.f32 %0, %1;" : "=f"(_abs_530) : "f"(rp_z_4.y)); + float _neg_529; + asm("neg.f32 %0, %1;" : "=f"(_neg_529) : "f"(_abs_530)); + float _exp_528 = __nv_expf(_neg_529); + float2 rp_e_4 = make_float2(_exp_525, _exp_528); float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_499 = approx_rcp(rp_den_4.x); - float _rcp_500 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_501 = make_float2(_rcp_499, _rcp_500); + float _rcp_531 = approx_rcp(rp_den_4.x); + float _rcp_532 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_533 = make_float2(_rcp_531, _rcp_532); float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_501)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_533)); float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_502 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_534 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_502)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_534)); u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_503 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_535 = make_float2(gacc[4], gacc[4]); float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_503), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_506; - asm("abs.f32 %0, %1;" : "=f"(_abs_506) : "f"(rp_z_5.x)); - float _neg_505; - asm("neg.f32 %0, %1;" : "=f"(_neg_505) : "f"(_abs_506)); - float _exp_504 = __nv_expf(_neg_505); - float _abs_509; - asm("abs.f32 %0, %1;" : "=f"(_abs_509) : "f"(rp_z_5.y)); - float _neg_508; - asm("neg.f32 %0, %1;" : "=f"(_neg_508) : "f"(_abs_509)); - float _exp_507 = __nv_expf(_neg_508); - float2 rp_e_5 = make_float2(_exp_504, _exp_507); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_535), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_538; + asm("abs.f32 %0, %1;" : "=f"(_abs_538) : "f"(rp_z_5.x)); + float _neg_537; + asm("neg.f32 %0, %1;" : "=f"(_neg_537) : "f"(_abs_538)); + float _exp_536 = __nv_expf(_neg_537); + float _abs_541; + asm("abs.f32 %0, %1;" : "=f"(_abs_541) : "f"(rp_z_5.y)); + float _neg_540; + asm("neg.f32 %0, %1;" : "=f"(_neg_540) : "f"(_abs_541)); + float _exp_539 = __nv_expf(_neg_540); + float2 rp_e_5 = make_float2(_exp_536, _exp_539); float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_510 = approx_rcp(rp_den_5.x); - float _rcp_511 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_512 = make_float2(_rcp_510, _rcp_511); + float _rcp_542 = approx_rcp(rp_den_5.x); + float _rcp_543 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_544 = make_float2(_rcp_542, _rcp_543); float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_512)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_544)); float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_513 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_545 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_513)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_545)); u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_514 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_546 = make_float2(gacc[4], gacc[4]); float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_514), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_517; - asm("abs.f32 %0, %1;" : "=f"(_abs_517) : "f"(rp_z_6.x)); - float _neg_516; - asm("neg.f32 %0, %1;" : "=f"(_neg_516) : "f"(_abs_517)); - float _exp_515 = __nv_expf(_neg_516); - float _abs_520; - asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_6.y)); - float _neg_519; - asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); - float _exp_518 = __nv_expf(_neg_519); - float2 rp_e_6 = make_float2(_exp_515, _exp_518); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_546), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_549; + asm("abs.f32 %0, %1;" : "=f"(_abs_549) : "f"(rp_z_6.x)); + float _neg_548; + asm("neg.f32 %0, %1;" : "=f"(_neg_548) : "f"(_abs_549)); + float _exp_547 = __nv_expf(_neg_548); + float _abs_552; + asm("abs.f32 %0, %1;" : "=f"(_abs_552) : "f"(rp_z_6.y)); + float _neg_551; + asm("neg.f32 %0, %1;" : "=f"(_neg_551) : "f"(_abs_552)); + float _exp_550 = __nv_expf(_neg_551); + float2 rp_e_6 = make_float2(_exp_547, _exp_550); float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_521 = approx_rcp(rp_den_6.x); - float _rcp_522 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_523 = make_float2(_rcp_521, _rcp_522); + float _rcp_553 = approx_rcp(rp_den_6.x); + float _rcp_554 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_555 = make_float2(_rcp_553, _rcp_554); float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_555)); float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_524 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_556 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_524)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_556)); u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4672,54 +4637,54 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_525; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_525) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_525; + uint32_t _prmt_557; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_557) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_557; int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_526 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_526)); - *(reinterpret_cast(_gst_526) + (0)) = pq_4[0]; + auto _gst_558 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_558)); + *(reinterpret_cast(_gst_558) + (0)) = pq_4[0]; } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_527; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_527) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_527; + uint32_t _prmt_559; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_559) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_559; int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_528 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_528)); - *(reinterpret_cast(_gst_528) + (0)) = pq_5[0]; + auto _gst_560 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_560)); + *(reinterpret_cast(_gst_560) + (0)) = pq_5[0]; } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_529; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_529) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_529; + uint32_t _prmt_561; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_561; int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_530 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_530)); - *(reinterpret_cast(_gst_530) + (0)) = pq_6[0]; + auto _gst_562 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_562)); + *(reinterpret_cast(_gst_562) + (0)) = pq_6[0]; } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_531; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_531) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_531; + uint32_t _prmt_563; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_563; int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_532 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_532)); - *(reinterpret_cast(_gst_532) + (0)) = pq_7[0]; + auto _gst_564 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_564)); + *(reinterpret_cast(_gst_564) + (0)) = pq_7[0]; } } int t_a_8 = 32 + 8 * th; @@ -4784,47 +4749,47 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_533; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_533) : "f"(s_a_0), "f"(nm_t_1)); + float _sub_565; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_565) : "f"(s_a_0), "f"(nm_t_1)); float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_533), "f"(pa_0)); - float _sub_534; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_534) : "f"(pa_0), "f"(nm_t_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_565), "f"(pa_0)); + float _sub_566; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_566) : "f"(pa_0), "f"(nm_t_1)); float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_534), "f"(s_a_0)); - float _abs_536; - asm("abs.f32 %0, %1;" : "=f"(_abs_536) : "f"(s_a_0)); - float _abs_537; - asm("abs.f32 %0, %1;" : "=f"(_abs_537) : "f"(pa_0)); - uint32_t _setp_535_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_535_bits) : "f"(_abs_536), "f"(_abs_537)); - bool _setp_535 = _setp_535_bits != 0; - float nm_err_1 = ((_setp_535) ? nm_es_1 : nm_ex_1); - float _add_538; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_538) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_538; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_566), "f"(s_a_0)); + float _abs_568; + asm("abs.f32 %0, %1;" : "=f"(_abs_568) : "f"(s_a_0)); + float _abs_569; + asm("abs.f32 %0, %1;" : "=f"(_abs_569) : "f"(pa_0)); + uint32_t _setp_567_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_567_bits) : "f"(_abs_568), "f"(_abs_569)); + bool _setp_567 = _setp_567_bits != 0; + float nm_err_1 = ((_setp_567) ? nm_es_1 : nm_ex_1); + float _add_570; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_570; s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_539; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_539) : "f"(s_b_0), "f"(nm_t_2)); + float _sub_571; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(s_b_0), "f"(nm_t_2)); float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_539), "f"(pb_0)); - float _sub_540; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_540) : "f"(pb_0), "f"(nm_t_2)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_571), "f"(pb_0)); + float _sub_572; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_572) : "f"(pb_0), "f"(nm_t_2)); float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_540), "f"(s_b_0)); - float _abs_542; - asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(s_b_0)); - float _abs_543; - asm("abs.f32 %0, %1;" : "=f"(_abs_543) : "f"(pb_0)); - uint32_t _setp_541_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_541_bits) : "f"(_abs_542), "f"(_abs_543)); - bool _setp_541 = _setp_541_bits != 0; - float nm_err_2 = ((_setp_541) ? nm_es_2 : nm_ex_2); - float _add_544; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_544) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_544; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_572), "f"(s_b_0)); + float _abs_574; + asm("abs.f32 %0, %1;" : "=f"(_abs_574) : "f"(s_b_0)); + float _abs_575; + asm("abs.f32 %0, %1;" : "=f"(_abs_575) : "f"(pb_0)); + uint32_t _setp_573_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_573_bits) : "f"(_abs_574), "f"(_abs_575)); + bool _setp_573 = _setp_573_bits != 0; + float nm_err_2 = ((_setp_573) ? nm_es_2 : nm_ex_2); + float _add_576; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_576; s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4832,8 +4797,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_545 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_545), "f"(off_acc[0]) : "memory"); + uint32_t _addr_577 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_577), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4859,342 +4824,155 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1292 = r_297[0]; - float gc_1293 = gn_1292 - 21.640425f; - int eb_1294 = 81920 + th * -32768; - int ew_1295 = eb_1294 >> 2; - int qbase_1296 = pool_f32_addr + (unsigned int)(eb_1294 + 12288 + dh_lane * 1024); - int sb8_1297 = 16 + 8 * th; - int gb8_1298 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1297 * 128); - int kb8_1299 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1297 * 128); - float dbp_1300[8]; - float _tmem_load_20[8]; - tmem_ld_x8(&_tmem_load_20[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); - float _tmem_load_21[8]; - tmem_ld_x8(&_tmem_load_21[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); - float _tmem_load_22[8]; - tmem_ld_x8(&_tmem_load_22[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); - float _tmem_load_23[8]; - tmem_ld_x8(&_tmem_load_23[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); - float _tmem_load_24[8]; - tmem_ld_x8(&_tmem_load_24[0], taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th)); - float btv_1301[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1301[0])), "=r"(*reinterpret_cast(&btv_1301[(0) + 1])), "=r"(*reinterpret_cast(&btv_1301[(0) + 2])), "=r"(*reinterpret_cast(&btv_1301[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1301[4])), "=r"(*reinterpret_cast(&btv_1301[(4) + 1])), "=r"(*reinterpret_cast(&btv_1301[(4) + 2])), "=r"(*reinterpret_cast(&btv_1301[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th) + 16))); + float r_276[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1 = r_276[0]; + float gc_1 = gn_1 - 21.640425f; + int eb_1 = 81920 + th * -32768; + int ew_3 = eb_1 >> 2; + int qbase_1 = pool_f32_addr + (unsigned int)(eb_1 + 12288 + dh_lane * 1024); + int sb8_1 = 16 + 8 * th; + int gb8_1 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1 * 128); + int kb8_1 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1 * 128); + int c_qa_0 = 64 + 8 * th; + int c_ka_0 = 80 + 8 * th; + int c_qd_0 = 96 + 8 * th; + int c_kd_0 = 112 + 8 * th; + int c_dk_0 = 16 + 8 * th; + int c_bp_0 = c_ka_0; + float s_run_1[1]; + s_run_1[0] = 0.0f; + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_3 = 7 - k_1; + float a_qd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_3))); + float a_kd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_0 + j_3))); + float a_dk_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_0 + j_3))); + float a_qa_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa_0 + j_3))); + float a_ka_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka_0 + j_3))); + float btr_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_1 + j_3)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_1 + j_3; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 * j_3 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_3 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 * j_3 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_3 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_3 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_3 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1 = __uint_as_float(r_279[0] << 16); + float q_m_1 = ((t_1 >= clen) ? 0.0f : q_t_1); + float dqf_1 = pool_f32[ew_3 + 128 * j_3 + d_lane]; + float dkf_1 = pool_f32[ew_3 + (1024 + 128 * j_3) + d_lane]; + float dgf_1 = pool_f32[ew_3 + (2048 + 128 * j_3) + d_lane]; + float bt_t_1 = btr_1[0]; + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float e_off_1 = approx_exp2(g_t_1 - gn_1); + float dq2_1 = __fmaf_rn(a_qa_1[0], e_off_1, a_qd_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(a_ka_1[0], e_off_1, a_kd_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + a_dk_1[0]; + float qe_8 = q_m_1 * e_dg_1; + float kp_20 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_1; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, a_qd_1[0], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_1) * a_kd_1[0]); + float dg_o_1 = __fmaf_rn(q_m_1, dq2_1, __fmaf_rn(dk2b_1 - a_dk_1[0], k_t_1, cr_8)) + dgf_1; + s_run_1[0] = s_run_1[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_3)), "f"(s_run_1[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_3)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(81920 - 32768 * th >> 2) + (128 * j_3 + d_lane)] = dq_o_1; + pool_f32[(81920 - 32768 * th >> 2) + (1024 + 128 * j_3 + d_lane)] = dk_o_1; + } + float tot_1 = s_run_1[0]; + float dbp_1[8]; + tmem_ld_x8(&dbp_1[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1302[1]; - s_run_1302[0] = 0.0f; - int t_8 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_1295 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_1295 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_1295 + 2944 + d_lane]; - float bt_t_8 = btv_1301[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_1292); - float e_dg_8 = approx_exp2(g_t_8 - gc_1293); - float dq2_8 = __fmaf_rn(_tmem_load_20[7], e_off_8, _tmem_load_22[7] * e_dg_8); - float dk2_8 = __fmaf_rn(_tmem_load_21[7], e_off_8, _tmem_load_23[7] * e_dg_8); - dbp_1300[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_24[7]; - float dg_o_8 = __fmaf_rn(q_t_1_16, dq2_8, (dk2b_8 - _tmem_load_24[7]) * k_t_8) + dgf_8; - s_run_1302[0] = s_run_1302[0] + dg_o_8; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_8; - pool_f32[(81920 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_8; - int t_9 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_1295 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_1295 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_1295 + 2816 + d_lane]; - float bt_t_9 = btv_1301[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_1292); - float e_dg_9 = approx_exp2(g_t_9 - gc_1293); - float dq2_9 = __fmaf_rn(_tmem_load_20[6], e_off_9, _tmem_load_22[6] * e_dg_9); - float dk2_9 = __fmaf_rn(_tmem_load_21[6], e_off_9, _tmem_load_23[6] * e_dg_9); - dbp_1300[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[6]; - float dg_o_9 = __fmaf_rn(q_t_1_17, dq2_9, (dk2b_9 - _tmem_load_24[6]) * k_t_9) + dgf_9; - s_run_1302[0] = s_run_1302[0] + dg_o_9; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_9; - pool_f32[(81920 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_9; - int t_10 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_1295 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_1295 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_1295 + 2688 + d_lane]; - float bt_t_10 = btv_1301[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_1292); - float e_dg_10 = approx_exp2(g_t_10 - gc_1293); - float dq2_10 = __fmaf_rn(_tmem_load_20[5], e_off_10, _tmem_load_22[5] * e_dg_10); - float dk2_10 = __fmaf_rn(_tmem_load_21[5], e_off_10, _tmem_load_23[5] * e_dg_10); - dbp_1300[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[5]; - float dg_o_10 = __fmaf_rn(q_t_1_18, dq2_10, (dk2b_10 - _tmem_load_24[5]) * k_t_10) + dgf_10; - s_run_1302[0] = s_run_1302[0] + dg_o_10; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_10; - pool_f32[(81920 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_10; - int t_11 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_1295 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_1295 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_1295 + 2560 + d_lane]; - float bt_t_11 = btv_1301[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_1292); - float e_dg_11 = approx_exp2(g_t_11 - gc_1293); - float dq2_11 = __fmaf_rn(_tmem_load_20[4], e_off_11, _tmem_load_22[4] * e_dg_11); - float dk2_11 = __fmaf_rn(_tmem_load_21[4], e_off_11, _tmem_load_23[4] * e_dg_11); - dbp_1300[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[4]; - float dg_o_11 = __fmaf_rn(q_t_1_19, dq2_11, (dk2b_11 - _tmem_load_24[4]) * k_t_11) + dgf_11; - s_run_1302[0] = s_run_1302[0] + dg_o_11; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_11; - pool_f32[(81920 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_11; - int t_12 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_1295 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_1295 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_1295 + 2432 + d_lane]; - float bt_t_12 = btv_1301[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_1292); - float e_dg_12 = approx_exp2(g_t_12 - gc_1293); - float dq2_12 = __fmaf_rn(_tmem_load_20[3], e_off_12, _tmem_load_22[3] * e_dg_12); - float dk2_12 = __fmaf_rn(_tmem_load_21[3], e_off_12, _tmem_load_23[3] * e_dg_12); - dbp_1300[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[3]; - float dg_o_12 = __fmaf_rn(q_t_1_20, dq2_12, (dk2b_12 - _tmem_load_24[3]) * k_t_12) + dgf_12; - s_run_1302[0] = s_run_1302[0] + dg_o_12; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_12; - pool_f32[(81920 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_12; - int t_13 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_1295 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_1295 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_1295 + 2304 + d_lane]; - float bt_t_13 = btv_1301[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_1292); - float e_dg_13 = approx_exp2(g_t_13 - gc_1293); - float dq2_13 = __fmaf_rn(_tmem_load_20[2], e_off_13, _tmem_load_22[2] * e_dg_13); - float dk2_13 = __fmaf_rn(_tmem_load_21[2], e_off_13, _tmem_load_23[2] * e_dg_13); - dbp_1300[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[2]; - float dg_o_13 = __fmaf_rn(q_t_1_21, dq2_13, (dk2b_13 - _tmem_load_24[2]) * k_t_13) + dgf_13; - s_run_1302[0] = s_run_1302[0] + dg_o_13; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_13; - pool_f32[(81920 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_13; - int t_14 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_1295 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_1295 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_1295 + 2176 + d_lane]; - float bt_t_14 = btv_1301[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_1292); - float e_dg_14 = approx_exp2(g_t_14 - gc_1293); - float dq2_14 = __fmaf_rn(_tmem_load_20[1], e_off_14, _tmem_load_22[1] * e_dg_14); - float dk2_14 = __fmaf_rn(_tmem_load_21[1], e_off_14, _tmem_load_23[1] * e_dg_14); - dbp_1300[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[1]; - float dg_o_14 = __fmaf_rn(q_t_1_22, dq2_14, (dk2b_14 - _tmem_load_24[1]) * k_t_14) + dgf_14; - s_run_1302[0] = s_run_1302[0] + dg_o_14; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_14; - pool_f32[(81920 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_14; - int t_15 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_1295 + d_lane]; - float dkf_15 = pool_f32[ew_1295 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_1295 + 2048 + d_lane]; - float bt_t_15 = btv_1301[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_1292); - float e_dg_15 = approx_exp2(g_t_15 - gc_1293); - float dq2_15 = __fmaf_rn(_tmem_load_20[0], e_off_15, _tmem_load_22[0] * e_dg_15); - float dk2_15 = __fmaf_rn(_tmem_load_21[0], e_off_15, _tmem_load_23[0] * e_dg_15); - dbp_1300[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[0]; - float dg_o_15 = __fmaf_rn(q_t_1_23, dq2_15, (dk2b_15 - _tmem_load_24[0]) * k_t_15) + dgf_15; - s_run_1302[0] = s_run_1302[0] + dg_o_15; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + d_lane] = dq_o_15; - pool_f32[(81920 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_15; - float tot_1471 = s_run_1302[0]; - int b4_1472 = lane_1 >> 4 & 1; - int b3_1473 = lane_1 >> 3 & 1; - int b2_1474 = lane_1 >> 2 & 1; - float v4_1475[4]; - float send_6 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_546 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_546; - float send_7 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_547 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_547; - float send_8 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_548 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_548; - float send_9 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_549 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_549; - float v2_1480[2]; - float send_10 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_550 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_550; - float send_11 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_551 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_551; - float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_552 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_552; - float _shfl_xor_sync_553 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_553; - float _shfl_xor_sync_554 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_554; + int b4_1 = lane_1 >> 4 & 1; + int b3_1 = lane_1 >> 3 & 1; + int b2_1 = lane_1 >> 2 & 1; + float v4_1[4]; + float send_4 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); + float _shfl_xor_sync_578 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_578; + float send_5 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); + float _shfl_xor_sync_579 = __shfl_xor_sync(0xFFFFFFFF, send_5, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_579; + float send_6 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); + float _shfl_xor_sync_580 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_580; + float send_7 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); + float _shfl_xor_sync_581 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_581; + float v2_16[2]; + float send2_2 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); + float _shfl_xor_sync_582 = __shfl_xor_sync(0xFFFFFFFF, send2_2, 8); + v2_16[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_582; + float send2_3 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); + float _shfl_xor_sync_583 = __shfl_xor_sync(0xFFFFFFFF, send2_3, 8); + v2_16[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_583; + float send1_0 = ((b2_1 == 0) ? v2_16[1] : v2_16[0]); + float _shfl_xor_sync_584 = __shfl_xor_sync(0xFFFFFFFF, send1_0, 4); + float v1_1 = ((b2_1 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_584; + float _shfl_xor_sync_585 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_585; + float _shfl_xor_sync_586 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_586; asm volatile("tcgen05.fence::before_thread_sync;"); - int ew_2 = 81920 - 32768 * th >> 2; + int ew_4 = 81920 - 32768 * th >> 2; float nz_1 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); asm volatile("barrier.sync 1, 256;" ::: "memory"); #pragma unroll 1 for (int i_2 = 0; i_2 < 2; i_2++) { - int j_2 = qd + 4 * i_2; - bool live_2 = 16 + 8 * th + j_2 < clen; - int _min_555 = ((16 + 8 * th + j_2) < (clen - 1) ? (16 + 8 * th + j_2) : (clen - 1)); - long long row_2 = ext0 + (long long)_min_555; + int j_4 = qd + 4 * i_2; + bool live_2 = 16 + 8 * th + j_4 < clen; + int _min_587 = ((16 + 8 * th + j_4) < (clen - 1) ? (16 + 8 * th + j_4) : (clen - 1)); + long long row_2 = ext0 + (long long)_min_587; float gq_2[4]; float gk_2[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gq_2[0])), "=r"(*reinterpret_cast(&gq_2[(0) + 1])), "=r"(*reinterpret_cast(&gq_2[(0) + 2])), "=r"(*reinterpret_cast(&gq_2[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_2 + 128 * j_2 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_4 + 128 * j_4 + 4 * lane_1)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gk_2[0])), "=r"(*reinterpret_cast(&gk_2[(0) + 1])), "=r"(*reinterpret_cast(&gk_2[(0) + 2])), "=r"(*reinterpret_cast(&gk_2[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_2 + 1024 + 128 * j_2 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_4 + 1024 + 128 * j_4 + 4 * lane_1)))); if (i_2 == 1) { mbarrier_arrive(epi_empty_addr + (2 + th) * 8); } @@ -5203,26 +4981,26 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float rq_2[1]; float rk_2[1]; { - uint2 _vld_556; - _vld_556 = *reinterpret_cast(q_raw + (row_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_556 = reinterpret_cast(&_vld_556); + uint2 _vld_588; + _vld_588 = *reinterpret_cast(q_raw + (row_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_588 = reinterpret_cast(&_vld_588); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xq_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_556[_pair]) << 16); - (&xq_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_556[_pair]) & 0xffff0000u); + (&xq_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_588[_pair]) << 16); + (&xq_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_588[_pair]) & 0xffff0000u); } } { rq_2[0] = *reinterpret_cast(qk_rstd + ((0 + row_2) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_557; - _vld_557 = *reinterpret_cast(k_raw + (row_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_557 = reinterpret_cast(&_vld_557); + uint2 _vld_589; + _vld_589 = *reinterpret_cast(k_raw + (row_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_589 = reinterpret_cast(&_vld_589); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xk_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_557[_pair]) << 16); - (&xk_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_557[_pair]) & 0xffff0000u); + (&xk_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_589[_pair]) << 16); + (&xk_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_589[_pair]) & 0xffff0000u); } } { @@ -5232,106 +5010,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_2[4]; float p_2[4]; y_2[0] = mul_f32_rn_noftz(xq_2[0], rq_2[0]); - float _fma_558; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_558) : "f"(gq_2[0]), "f"(y_2[0]), "f"(nz_1)); - p_2[0] = _fma_558; + float _fma_590; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_590) : "f"(gq_2[0]), "f"(y_2[0]), "f"(nz_1)); + p_2[0] = _fma_590; y_2[1] = mul_f32_rn_noftz(xq_2[1], rq_2[0]); - float _fma_559; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_559) : "f"(gq_2[1]), "f"(y_2[1]), "f"(nz_1)); - p_2[1] = _fma_559; + float _fma_591; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_591) : "f"(gq_2[1]), "f"(y_2[1]), "f"(nz_1)); + p_2[1] = _fma_591; y_2[2] = mul_f32_rn_noftz(xq_2[2], rq_2[0]); - float _fma_560; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_560) : "f"(gq_2[2]), "f"(y_2[2]), "f"(nz_1)); - p_2[2] = _fma_560; + float _fma_592; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_592) : "f"(gq_2[2]), "f"(y_2[2]), "f"(nz_1)); + p_2[2] = _fma_592; y_2[3] = mul_f32_rn_noftz(xq_2[3], rq_2[0]); - float _fma_561; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_561) : "f"(gq_2[3]), "f"(y_2[3]), "f"(nz_1)); - p_2[3] = _fma_561; - float _shfl_xor_sync_563 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 4); - float _add_562; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_562) : "f"(p_2[0]), "f"(_shfl_xor_sync_563)); - p_2[0] = _add_562; - float _shfl_xor_sync_565 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 4); - float _add_564; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_564) : "f"(p_2[1]), "f"(_shfl_xor_sync_565)); - p_2[1] = _add_564; - float _shfl_xor_sync_567 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 4); - float _add_566; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_566) : "f"(p_2[2]), "f"(_shfl_xor_sync_567)); - p_2[2] = _add_566; - float _shfl_xor_sync_569 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 4); - float _add_568; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_568) : "f"(p_2[3]), "f"(_shfl_xor_sync_569)); - p_2[3] = _add_568; - float _shfl_xor_sync_571 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 2); - float _add_570; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(p_2[0]), "f"(_shfl_xor_sync_571)); - p_2[0] = _add_570; - float _shfl_xor_sync_573 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 2); - float _add_572; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_572) : "f"(p_2[1]), "f"(_shfl_xor_sync_573)); - p_2[1] = _add_572; - float _shfl_xor_sync_575 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 2); - float _add_574; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(p_2[2]), "f"(_shfl_xor_sync_575)); - p_2[2] = _add_574; - float _shfl_xor_sync_577 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 2); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(p_2[3]), "f"(_shfl_xor_sync_577)); - p_2[3] = _add_576; - float _shfl_xor_sync_579 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 1); - float _add_578; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_578) : "f"(p_2[0]), "f"(_shfl_xor_sync_579)); - p_2[0] = _add_578; - float _shfl_xor_sync_581 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 1); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(p_2[1]), "f"(_shfl_xor_sync_581)); - p_2[1] = _add_580; - float _shfl_xor_sync_583 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 1); - float _add_582; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_582) : "f"(p_2[2]), "f"(_shfl_xor_sync_583)); - p_2[2] = _add_582; - float _shfl_xor_sync_585 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 1); - float _add_584; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(p_2[3]), "f"(_shfl_xor_sync_585)); - p_2[3] = _add_584; - float _add_586; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_586) : "f"(p_2[0]), "f"(p_2[2])); - float _add_587; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_587) : "f"(p_2[1]), "f"(p_2[3])); + float _fma_593; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_593) : "f"(gq_2[3]), "f"(y_2[3]), "f"(nz_1)); + p_2[3] = _fma_593; + float _shfl_xor_sync_595 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 4); + float _add_594; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_594) : "f"(p_2[0]), "f"(_shfl_xor_sync_595)); + p_2[0] = _add_594; + float _shfl_xor_sync_597 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 4); + float _add_596; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_596) : "f"(p_2[1]), "f"(_shfl_xor_sync_597)); + p_2[1] = _add_596; + float _shfl_xor_sync_599 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 4); + float _add_598; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_598) : "f"(p_2[2]), "f"(_shfl_xor_sync_599)); + p_2[2] = _add_598; + float _shfl_xor_sync_601 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 4); + float _add_600; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_600) : "f"(p_2[3]), "f"(_shfl_xor_sync_601)); + p_2[3] = _add_600; + float _shfl_xor_sync_603 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 2); + float _add_602; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_602) : "f"(p_2[0]), "f"(_shfl_xor_sync_603)); + p_2[0] = _add_602; + float _shfl_xor_sync_605 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 2); + float _add_604; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_604) : "f"(p_2[1]), "f"(_shfl_xor_sync_605)); + p_2[1] = _add_604; + float _shfl_xor_sync_607 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 2); + float _add_606; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_606) : "f"(p_2[2]), "f"(_shfl_xor_sync_607)); + p_2[2] = _add_606; + float _shfl_xor_sync_609 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 2); + float _add_608; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_608) : "f"(p_2[3]), "f"(_shfl_xor_sync_609)); + p_2[3] = _add_608; + float _shfl_xor_sync_611 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 1); + float _add_610; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_610) : "f"(p_2[0]), "f"(_shfl_xor_sync_611)); + p_2[0] = _add_610; + float _shfl_xor_sync_613 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 1); + float _add_612; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_612) : "f"(p_2[1]), "f"(_shfl_xor_sync_613)); + p_2[1] = _add_612; + float _shfl_xor_sync_615 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 1); + float _add_614; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_614) : "f"(p_2[2]), "f"(_shfl_xor_sync_615)); + p_2[2] = _add_614; + float _shfl_xor_sync_617 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 1); + float _add_616; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_616) : "f"(p_2[3]), "f"(_shfl_xor_sync_617)); + p_2[3] = _add_616; + float _add_618; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_618) : "f"(p_2[0]), "f"(p_2[2])); + float _add_619; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_619) : "f"(p_2[1]), "f"(p_2[3])); float sq_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2) : "f"(_add_586), "f"(_add_587)); - float _shfl_sync_590 = __shfl_sync(0xFFFFFFFF, sq_2, 0); - float _shfl_sync_591 = __shfl_sync(0xFFFFFFFF, sq_2, 8); - float _add_589; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_589) : "f"(_shfl_sync_590), "f"(_shfl_sync_591)); - float _shfl_sync_592 = __shfl_sync(0xFFFFFFFF, sq_2, 16); - float _add_588; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_588) : "f"(_add_589), "f"(_shfl_sync_592)); - float _shfl_sync_593 = __shfl_sync(0xFFFFFFFF, sq_2, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2) : "f"(_add_618), "f"(_add_619)); + float _shfl_sync_622 = __shfl_sync(0xFFFFFFFF, sq_2, 0); + float _shfl_sync_623 = __shfl_sync(0xFFFFFFFF, sq_2, 8); + float _add_621; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_621) : "f"(_shfl_sync_622), "f"(_shfl_sync_623)); + float _shfl_sync_624 = __shfl_sync(0xFFFFFFFF, sq_2, 16); + float _add_620; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_620) : "f"(_add_621), "f"(_shfl_sync_624)); + float _shfl_sync_625 = __shfl_sync(0xFFFFFFFF, sq_2, 24); float dot_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_588), "f"(_shfl_sync_593)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_620), "f"(_shfl_sync_625)); float o_2[4]; - float _fma_595; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_595) : "f"(y_2[0]), "f"(dot_2), "f"(nz_1)); - float _sub_594; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_594) : "f"(gq_2[0]), "f"(_fma_595)); - o_2[0] = mul_f32_rn_noftz(rq_2[0], _sub_594); - float _fma_597; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_597) : "f"(y_2[1]), "f"(dot_2), "f"(nz_1)); - float _sub_596; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_596) : "f"(gq_2[1]), "f"(_fma_597)); - o_2[1] = mul_f32_rn_noftz(rq_2[0], _sub_596); - float _fma_599; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_599) : "f"(y_2[2]), "f"(dot_2), "f"(nz_1)); - float _sub_598; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_598) : "f"(gq_2[2]), "f"(_fma_599)); - o_2[2] = mul_f32_rn_noftz(rq_2[0], _sub_598); - float _fma_601; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_601) : "f"(y_2[3]), "f"(dot_2), "f"(nz_1)); - float _sub_600; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_600) : "f"(gq_2[3]), "f"(_fma_601)); - o_2[3] = mul_f32_rn_noftz(rq_2[0], _sub_600); + float _fma_627; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_627) : "f"(y_2[0]), "f"(dot_2), "f"(nz_1)); + float _sub_626; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_626) : "f"(gq_2[0]), "f"(_fma_627)); + o_2[0] = mul_f32_rn_noftz(rq_2[0], _sub_626); + float _fma_629; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_629) : "f"(y_2[1]), "f"(dot_2), "f"(nz_1)); + float _sub_628; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_628) : "f"(gq_2[1]), "f"(_fma_629)); + o_2[1] = mul_f32_rn_noftz(rq_2[0], _sub_628); + float _fma_631; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_631) : "f"(y_2[2]), "f"(dot_2), "f"(nz_1)); + float _sub_630; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_630) : "f"(gq_2[2]), "f"(_fma_631)); + o_2[2] = mul_f32_rn_noftz(rq_2[0], _sub_630); + float _fma_633; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_633) : "f"(y_2[3]), "f"(dot_2), "f"(nz_1)); + float _sub_632; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_632) : "f"(gq_2[3]), "f"(_fma_633)); + o_2[3] = mul_f32_rn_noftz(rq_2[0], _sub_632); unsigned int ob_2[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -5342,106 +5120,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_2_1[4]; float p_2_1[4]; y_2_1[0] = mul_f32_rn_noftz(xk_2[0], rk_2[0]); - float _fma_602; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_602) : "f"(gk_2[0]), "f"(y_2_1[0]), "f"(nz_1)); - p_2_1[0] = _fma_602; + float _fma_634; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_634) : "f"(gk_2[0]), "f"(y_2_1[0]), "f"(nz_1)); + p_2_1[0] = _fma_634; y_2_1[1] = mul_f32_rn_noftz(xk_2[1], rk_2[0]); - float _fma_603; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_603) : "f"(gk_2[1]), "f"(y_2_1[1]), "f"(nz_1)); - p_2_1[1] = _fma_603; + float _fma_635; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_635) : "f"(gk_2[1]), "f"(y_2_1[1]), "f"(nz_1)); + p_2_1[1] = _fma_635; y_2_1[2] = mul_f32_rn_noftz(xk_2[2], rk_2[0]); - float _fma_604; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_604) : "f"(gk_2[2]), "f"(y_2_1[2]), "f"(nz_1)); - p_2_1[2] = _fma_604; + float _fma_636; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_636) : "f"(gk_2[2]), "f"(y_2_1[2]), "f"(nz_1)); + p_2_1[2] = _fma_636; y_2_1[3] = mul_f32_rn_noftz(xk_2[3], rk_2[0]); - float _fma_605; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_605) : "f"(gk_2[3]), "f"(y_2_1[3]), "f"(nz_1)); - p_2_1[3] = _fma_605; - float _shfl_xor_sync_607 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 4); - float _add_606; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_606) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_607)); - p_2_1[0] = _add_606; - float _shfl_xor_sync_609 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 4); - float _add_608; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_608) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_609)); - p_2_1[1] = _add_608; - float _shfl_xor_sync_611 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 4); - float _add_610; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_610) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_611)); - p_2_1[2] = _add_610; - float _shfl_xor_sync_613 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 4); - float _add_612; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_612) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_613)); - p_2_1[3] = _add_612; - float _shfl_xor_sync_615 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 2); - float _add_614; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_614) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_615)); - p_2_1[0] = _add_614; - float _shfl_xor_sync_617 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 2); - float _add_616; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_616) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_617)); - p_2_1[1] = _add_616; - float _shfl_xor_sync_619 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 2); - float _add_618; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_618) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_619)); - p_2_1[2] = _add_618; - float _shfl_xor_sync_621 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 2); - float _add_620; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_620) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_621)); - p_2_1[3] = _add_620; - float _shfl_xor_sync_623 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 1); - float _add_622; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_622) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_623)); - p_2_1[0] = _add_622; - float _shfl_xor_sync_625 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 1); - float _add_624; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_624) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_625)); - p_2_1[1] = _add_624; - float _shfl_xor_sync_627 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 1); - float _add_626; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_626) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_627)); - p_2_1[2] = _add_626; - float _shfl_xor_sync_629 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 1); - float _add_628; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_628) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_629)); - p_2_1[3] = _add_628; - float _add_630; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_630) : "f"(p_2_1[0]), "f"(p_2_1[2])); - float _add_631; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_631) : "f"(p_2_1[1]), "f"(p_2_1[3])); + float _fma_637; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_637) : "f"(gk_2[3]), "f"(y_2_1[3]), "f"(nz_1)); + p_2_1[3] = _fma_637; + float _shfl_xor_sync_639 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 4); + float _add_638; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_638) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_639)); + p_2_1[0] = _add_638; + float _shfl_xor_sync_641 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 4); + float _add_640; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_640) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_641)); + p_2_1[1] = _add_640; + float _shfl_xor_sync_643 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 4); + float _add_642; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_642) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_643)); + p_2_1[2] = _add_642; + float _shfl_xor_sync_645 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 4); + float _add_644; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_644) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_645)); + p_2_1[3] = _add_644; + float _shfl_xor_sync_647 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 2); + float _add_646; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_646) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_647)); + p_2_1[0] = _add_646; + float _shfl_xor_sync_649 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 2); + float _add_648; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_648) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_649)); + p_2_1[1] = _add_648; + float _shfl_xor_sync_651 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 2); + float _add_650; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_650) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_651)); + p_2_1[2] = _add_650; + float _shfl_xor_sync_653 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 2); + float _add_652; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_652) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_653)); + p_2_1[3] = _add_652; + float _shfl_xor_sync_655 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 1); + float _add_654; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_654) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_655)); + p_2_1[0] = _add_654; + float _shfl_xor_sync_657 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 1); + float _add_656; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_656) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_657)); + p_2_1[1] = _add_656; + float _shfl_xor_sync_659 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 1); + float _add_658; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_658) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_659)); + p_2_1[2] = _add_658; + float _shfl_xor_sync_661 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 1); + float _add_660; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_660) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_661)); + p_2_1[3] = _add_660; + float _add_662; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_662) : "f"(p_2_1[0]), "f"(p_2_1[2])); + float _add_663; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_663) : "f"(p_2_1[1]), "f"(p_2_1[3])); float sq_2_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2_1) : "f"(_add_630), "f"(_add_631)); - float _shfl_sync_634 = __shfl_sync(0xFFFFFFFF, sq_2_1, 0); - float _shfl_sync_635 = __shfl_sync(0xFFFFFFFF, sq_2_1, 8); - float _add_633; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_633) : "f"(_shfl_sync_634), "f"(_shfl_sync_635)); - float _shfl_sync_636 = __shfl_sync(0xFFFFFFFF, sq_2_1, 16); - float _add_632; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_632) : "f"(_add_633), "f"(_shfl_sync_636)); - float _shfl_sync_637 = __shfl_sync(0xFFFFFFFF, sq_2_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2_1) : "f"(_add_662), "f"(_add_663)); + float _shfl_sync_666 = __shfl_sync(0xFFFFFFFF, sq_2_1, 0); + float _shfl_sync_667 = __shfl_sync(0xFFFFFFFF, sq_2_1, 8); + float _add_665; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_665) : "f"(_shfl_sync_666), "f"(_shfl_sync_667)); + float _shfl_sync_668 = __shfl_sync(0xFFFFFFFF, sq_2_1, 16); + float _add_664; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_664) : "f"(_add_665), "f"(_shfl_sync_668)); + float _shfl_sync_669 = __shfl_sync(0xFFFFFFFF, sq_2_1, 24); float dot_2_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2_1) : "f"(_add_632), "f"(_shfl_sync_637)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2_1) : "f"(_add_664), "f"(_shfl_sync_669)); float o_2_1[4]; - float _fma_639; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_639) : "f"(y_2_1[0]), "f"(dot_2_1), "f"(nz_1)); - float _sub_638; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_638) : "f"(gk_2[0]), "f"(_fma_639)); - o_2_1[0] = mul_f32_rn_noftz(rk_2[0], _sub_638); - float _fma_641; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_641) : "f"(y_2_1[1]), "f"(dot_2_1), "f"(nz_1)); - float _sub_640; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_640) : "f"(gk_2[1]), "f"(_fma_641)); - o_2_1[1] = mul_f32_rn_noftz(rk_2[0], _sub_640); - float _fma_643; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_643) : "f"(y_2_1[2]), "f"(dot_2_1), "f"(nz_1)); - float _sub_642; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_642) : "f"(gk_2[2]), "f"(_fma_643)); - o_2_1[2] = mul_f32_rn_noftz(rk_2[0], _sub_642); - float _fma_645; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_645) : "f"(y_2_1[3]), "f"(dot_2_1), "f"(nz_1)); - float _sub_644; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_644) : "f"(gk_2[3]), "f"(_fma_645)); - o_2_1[3] = mul_f32_rn_noftz(rk_2[0], _sub_644); + float _fma_671; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_671) : "f"(y_2_1[0]), "f"(dot_2_1), "f"(nz_1)); + float _sub_670; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_670) : "f"(gk_2[0]), "f"(_fma_671)); + o_2_1[0] = mul_f32_rn_noftz(rk_2[0], _sub_670); + float _fma_673; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_673) : "f"(y_2_1[1]), "f"(dot_2_1), "f"(nz_1)); + float _sub_672; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_672) : "f"(gk_2[1]), "f"(_fma_673)); + o_2_1[1] = mul_f32_rn_noftz(rk_2[0], _sub_672); + float _fma_675; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_675) : "f"(y_2_1[2]), "f"(dot_2_1), "f"(nz_1)); + float _sub_674; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_674) : "f"(gk_2[2]), "f"(_fma_675)); + o_2_1[2] = mul_f32_rn_noftz(rk_2[0], _sub_674); + float _fma_677; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_677) : "f"(y_2_1[3]), "f"(dot_2_1), "f"(nz_1)); + float _sub_676; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_676) : "f"(gk_2[3]), "f"(_fma_677)); + o_2_1[3] = mul_f32_rn_noftz(rk_2[0], _sub_676); unsigned int ob_2_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -5454,8 +5232,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_646 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_646), "f"(tot_1471) : "memory"); + uint32_t _addr_678 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_678), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5473,8 +5251,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other_0[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other_0[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi_1 = ((th == 1) ? tot_1471 : t_other_0[0]); - float t_lo_1 = ((th == 1) ? t_other_0[0] : tot_1471); + float t_hi_1 = ((th == 1) ? tot_1 : t_other_0[0]); + float t_lo_1 = ((th == 1) ? t_other_0[0] : tot_1); float offset_1 = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi_1); float sfx_1[8]; tmem_ld_x8(&sfx_1[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); @@ -5487,132 +5265,132 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_647 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_679 = make_float2(gacc[4], gacc[4]); float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_647), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_650; - asm("abs.f32 %0, %1;" : "=f"(_abs_650) : "f"(rp_z_7.x)); - float _neg_649; - asm("neg.f32 %0, %1;" : "=f"(_neg_649) : "f"(_abs_650)); - float _exp_648 = __nv_expf(_neg_649); - float _abs_653; - asm("abs.f32 %0, %1;" : "=f"(_abs_653) : "f"(rp_z_7.y)); - float _neg_652; - asm("neg.f32 %0, %1;" : "=f"(_neg_652) : "f"(_abs_653)); - float _exp_651 = __nv_expf(_neg_652); - float2 rp_e_7 = make_float2(_exp_648, _exp_651); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_679), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_682; + asm("abs.f32 %0, %1;" : "=f"(_abs_682) : "f"(rp_z_7.x)); + float _neg_681; + asm("neg.f32 %0, %1;" : "=f"(_neg_681) : "f"(_abs_682)); + float _exp_680 = __nv_expf(_neg_681); + float _abs_685; + asm("abs.f32 %0, %1;" : "=f"(_abs_685) : "f"(rp_z_7.y)); + float _neg_684; + asm("neg.f32 %0, %1;" : "=f"(_neg_684) : "f"(_abs_685)); + float _exp_683 = __nv_expf(_neg_684); + float2 rp_e_7 = make_float2(_exp_680, _exp_683); float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_654 = approx_rcp(rp_den_7.x); - float _rcp_655 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_656 = make_float2(_rcp_654, _rcp_655); + float _rcp_686 = approx_rcp(rp_den_7.x); + float _rcp_687 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_688 = make_float2(_rcp_686, _rcp_687); float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_656)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_688)); float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_657 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_689 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_657)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_689)); u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_658 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_690 = make_float2(gacc[4], gacc[4]); float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_658), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_661; - asm("abs.f32 %0, %1;" : "=f"(_abs_661) : "f"(rp_z_8.x)); - float _neg_660; - asm("neg.f32 %0, %1;" : "=f"(_neg_660) : "f"(_abs_661)); - float _exp_659 = __nv_expf(_neg_660); - float _abs_664; - asm("abs.f32 %0, %1;" : "=f"(_abs_664) : "f"(rp_z_8.y)); - float _neg_663; - asm("neg.f32 %0, %1;" : "=f"(_neg_663) : "f"(_abs_664)); - float _exp_662 = __nv_expf(_neg_663); - float2 rp_e_8 = make_float2(_exp_659, _exp_662); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_690), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_693; + asm("abs.f32 %0, %1;" : "=f"(_abs_693) : "f"(rp_z_8.x)); + float _neg_692; + asm("neg.f32 %0, %1;" : "=f"(_neg_692) : "f"(_abs_693)); + float _exp_691 = __nv_expf(_neg_692); + float _abs_696; + asm("abs.f32 %0, %1;" : "=f"(_abs_696) : "f"(rp_z_8.y)); + float _neg_695; + asm("neg.f32 %0, %1;" : "=f"(_neg_695) : "f"(_abs_696)); + float _exp_694 = __nv_expf(_neg_695); + float2 rp_e_8 = make_float2(_exp_691, _exp_694); float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_665 = approx_rcp(rp_den_8.x); - float _rcp_666 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_667 = make_float2(_rcp_665, _rcp_666); + float _rcp_697 = approx_rcp(rp_den_8.x); + float _rcp_698 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_699 = make_float2(_rcp_697, _rcp_698); float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_667)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_699)); float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_668 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_700 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_668)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_700)); u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_669 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_701 = make_float2(gacc[4], gacc[4]); float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_669), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_672; - asm("abs.f32 %0, %1;" : "=f"(_abs_672) : "f"(rp_z_9.x)); - float _neg_671; - asm("neg.f32 %0, %1;" : "=f"(_neg_671) : "f"(_abs_672)); - float _exp_670 = __nv_expf(_neg_671); - float _abs_675; - asm("abs.f32 %0, %1;" : "=f"(_abs_675) : "f"(rp_z_9.y)); - float _neg_674; - asm("neg.f32 %0, %1;" : "=f"(_neg_674) : "f"(_abs_675)); - float _exp_673 = __nv_expf(_neg_674); - float2 rp_e_9 = make_float2(_exp_670, _exp_673); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_701), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_704; + asm("abs.f32 %0, %1;" : "=f"(_abs_704) : "f"(rp_z_9.x)); + float _neg_703; + asm("neg.f32 %0, %1;" : "=f"(_neg_703) : "f"(_abs_704)); + float _exp_702 = __nv_expf(_neg_703); + float _abs_707; + asm("abs.f32 %0, %1;" : "=f"(_abs_707) : "f"(rp_z_9.y)); + float _neg_706; + asm("neg.f32 %0, %1;" : "=f"(_neg_706) : "f"(_abs_707)); + float _exp_705 = __nv_expf(_neg_706); + float2 rp_e_9 = make_float2(_exp_702, _exp_705); float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_676 = approx_rcp(rp_den_9.x); - float _rcp_677 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_678 = make_float2(_rcp_676, _rcp_677); + float _rcp_708 = approx_rcp(rp_den_9.x); + float _rcp_709 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_710 = make_float2(_rcp_708, _rcp_709); float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_678)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_710)); float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_679 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_711 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_679)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_711)); u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_680 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_712 = make_float2(gacc[4], gacc[4]); float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_680), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_683; - asm("abs.f32 %0, %1;" : "=f"(_abs_683) : "f"(rp_z_10.x)); - float _neg_682; - asm("neg.f32 %0, %1;" : "=f"(_neg_682) : "f"(_abs_683)); - float _exp_681 = __nv_expf(_neg_682); - float _abs_686; - asm("abs.f32 %0, %1;" : "=f"(_abs_686) : "f"(rp_z_10.y)); - float _neg_685; - asm("neg.f32 %0, %1;" : "=f"(_neg_685) : "f"(_abs_686)); - float _exp_684 = __nv_expf(_neg_685); - float2 rp_e_10 = make_float2(_exp_681, _exp_684); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_712), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_715; + asm("abs.f32 %0, %1;" : "=f"(_abs_715) : "f"(rp_z_10.x)); + float _neg_714; + asm("neg.f32 %0, %1;" : "=f"(_neg_714) : "f"(_abs_715)); + float _exp_713 = __nv_expf(_neg_714); + float _abs_718; + asm("abs.f32 %0, %1;" : "=f"(_abs_718) : "f"(rp_z_10.y)); + float _neg_717; + asm("neg.f32 %0, %1;" : "=f"(_neg_717) : "f"(_abs_718)); + float _exp_716 = __nv_expf(_neg_717); + float2 rp_e_10 = make_float2(_exp_713, _exp_716); float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_687 = approx_rcp(rp_den_10.x); - float _rcp_688 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_689 = make_float2(_rcp_687, _rcp_688); + float _rcp_719 = approx_rcp(rp_den_10.x); + float _rcp_720 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_721 = make_float2(_rcp_719, _rcp_720); float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_689)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_721)); float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_690 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_722 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_690)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_722)); u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -5627,54 +5405,54 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_691; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_691) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_691; + uint32_t _prmt_723; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_723) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_723; int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_692 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_692)); - *(reinterpret_cast(_gst_692) + (0)) = pq_8[0]; + auto _gst_724 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_724)); + *(reinterpret_cast(_gst_724) + (0)) = pq_8[0]; } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_693; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_693) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_693; + uint32_t _prmt_725; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_725) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_725; int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_694 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_694)); - *(reinterpret_cast(_gst_694) + (0)) = pq_9[0]; + auto _gst_726 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_726)); + *(reinterpret_cast(_gst_726) + (0)) = pq_9[0]; } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_695; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_695) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_695; + uint32_t _prmt_727; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_727) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_727; int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_696 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_696)); - *(reinterpret_cast(_gst_696) + (0)) = pq_10[0]; + auto _gst_728 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_728)); + *(reinterpret_cast(_gst_728) + (0)) = pq_10[0]; } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_697; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_697) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_697; + uint32_t _prmt_729; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_729) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_729; int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_698 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_698)); - *(reinterpret_cast(_gst_698) + (0)) = pq_11[0]; + auto _gst_730 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_730)); + *(reinterpret_cast(_gst_730) + (0)) = pq_11[0]; } } int t_a_16 = 16 + 8 * th; @@ -5739,47 +5517,47 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_699; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_699) : "f"(s_a_1), "f"(nm_t_3)); + float _sub_731; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_731) : "f"(s_a_1), "f"(nm_t_3)); float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_699), "f"(pa_1)); - float _sub_700; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_700) : "f"(pa_1), "f"(nm_t_3)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_731), "f"(pa_1)); + float _sub_732; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_732) : "f"(pa_1), "f"(nm_t_3)); float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_700), "f"(s_a_1)); - float _abs_702; - asm("abs.f32 %0, %1;" : "=f"(_abs_702) : "f"(s_a_1)); - float _abs_703; - asm("abs.f32 %0, %1;" : "=f"(_abs_703) : "f"(pa_1)); - uint32_t _setp_701_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_701_bits) : "f"(_abs_702), "f"(_abs_703)); - bool _setp_701 = _setp_701_bits != 0; - float nm_err_3 = ((_setp_701) ? nm_es_3 : nm_ex_3); - float _add_704; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_704) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_704; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_732), "f"(s_a_1)); + float _abs_734; + asm("abs.f32 %0, %1;" : "=f"(_abs_734) : "f"(s_a_1)); + float _abs_735; + asm("abs.f32 %0, %1;" : "=f"(_abs_735) : "f"(pa_1)); + uint32_t _setp_733_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_733_bits) : "f"(_abs_734), "f"(_abs_735)); + bool _setp_733 = _setp_733_bits != 0; + float nm_err_3 = ((_setp_733) ? nm_es_3 : nm_ex_3); + float _add_736; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_736) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_736; s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_705; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_705) : "f"(s_b_1), "f"(nm_t_4)); + float _sub_737; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_737) : "f"(s_b_1), "f"(nm_t_4)); float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_705), "f"(pb_1)); - float _sub_706; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_706) : "f"(pb_1), "f"(nm_t_4)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_737), "f"(pb_1)); + float _sub_738; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_738) : "f"(pb_1), "f"(nm_t_4)); float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_706), "f"(s_b_1)); - float _abs_708; - asm("abs.f32 %0, %1;" : "=f"(_abs_708) : "f"(s_b_1)); - float _abs_709; - asm("abs.f32 %0, %1;" : "=f"(_abs_709) : "f"(pb_1)); - uint32_t _setp_707_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_707_bits) : "f"(_abs_708), "f"(_abs_709)); - bool _setp_707 = _setp_707_bits != 0; - float nm_err_4 = ((_setp_707) ? nm_es_4 : nm_ex_4); - float _add_710; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_710) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_710; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_738), "f"(s_b_1)); + float _abs_740; + asm("abs.f32 %0, %1;" : "=f"(_abs_740) : "f"(s_b_1)); + float _abs_741; + asm("abs.f32 %0, %1;" : "=f"(_abs_741) : "f"(pb_1)); + uint32_t _setp_739_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_739_bits) : "f"(_abs_740), "f"(_abs_741)); + bool _setp_739 = _setp_739_bits != 0; + float nm_err_4 = ((_setp_739) ? nm_es_4 : nm_ex_4); + float _add_742; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_742) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_742; s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5787,8 +5565,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_711 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_711), "f"(off_acc[0]) : "memory"); + uint32_t _addr_743 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_743), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5812,362 +5590,166 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1514 = r_322[0]; - float gc_1515 = gn_1514 - 21.640425f; - int eb_1516 = 98304 + th * -32768; - int ew_1517 = eb_1516 >> 2; - int qbase_1518 = pool_f32_addr + (unsigned int)(eb_1516 + 12288 + dh_lane * 1024); - int sb8_1519 = 8 * th; - int gb8_1520 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1519 * 128); - int kb8_1521 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1519 * 128); - float dbp_1522[8]; - float _tmem_load_26[8]; - tmem_ld_x8(&_tmem_load_26[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float _tmem_load_27[8]; - tmem_ld_x8(&_tmem_load_27[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float _tmem_load_28[8]; - tmem_ld_x8(&_tmem_load_28[0], taddr + (unsigned int)row_base + (unsigned int)(8 * th)); - float btv_1523[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1523[0])), "=r"(*reinterpret_cast(&btv_1523[(0) + 1])), "=r"(*reinterpret_cast(&btv_1523[(0) + 2])), "=r"(*reinterpret_cast(&btv_1523[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1523[4])), "=r"(*reinterpret_cast(&btv_1523[(4) + 1])), "=r"(*reinterpret_cast(&btv_1523[(4) + 2])), "=r"(*reinterpret_cast(&btv_1523[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th) + 16))); + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_2 = r_280[0]; + float gc_2 = gn_2 - 21.640425f; + int eb_2 = 98304 + th * -32768; + int ew_5 = eb_2 >> 2; + int qbase_2 = pool_f32_addr + (unsigned int)(eb_2 + 12288 + dh_lane * 1024); + int sb8_2 = 8 * th; + int gb8_2 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_2 * 128); + int kb8_2 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_2 * 128); + int c_qa_1 = 8 * th; + int c_ka_1 = 8 * th; + int c_qd_1 = 128 + 8 * th; + int c_kd_1 = 144 + 8 * th; + int c_dk_1 = 8 * th; + int c_bp_1 = c_kd_1; + float s_run_2[1]; + s_run_2[0] = 0.0f; + #pragma unroll 1 + for (int k_2 = 0; k_2 < 8; k_2++) { + int j_5 = 7 - k_2; + float a_qd_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_1 + j_5))); + float a_kd_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_1 + j_5))); + float a_dk_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_1 + j_5))); + float btr_2[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_2[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_2 + j_5)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_2 = sb8_2 + j_5; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 * j_5 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_5 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 * j_5 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_5 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 * j_5 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_5 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_283[0] << 16); + float q_m_2 = ((t_2 >= clen) ? 0.0f : q_t_2); + float dqf_2 = pool_f32[ew_5 + 128 * j_5 + d_lane]; + float dkf_2 = pool_f32[ew_5 + (1024 + 128 * j_5) + d_lane]; + float dgf_2 = pool_f32[ew_5 + (2048 + 128 * j_5) + d_lane]; + float bt_t_2 = btr_2[0]; + float e_dg_2 = approx_exp2(g_t_2 - gc_2); + float dq2_2 = a_qd_2[0] * e_dg_2; + float dk2_2 = a_kd_2[0] * e_dg_2; + float dbv_2[1]; + dbv_2[0] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + a_dk_2[0]; + float qe_9 = q_m_2 * e_dg_2; + float kp_21 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, a_qd_2[0], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_2) * a_kd_2[0]); + float dg_o_2 = __fmaf_rn(q_m_2, dq2_2, __fmaf_rn(dk2b_2 - a_dk_2[0], k_t_2, cr_9)) + dgf_2; + s_run_2[0] = s_run_2[0] + dg_o_2; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_1 + j_5)), "f"(s_run_2[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_1 + j_5)), "f"(dbv_2[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(98304 - 32768 * th >> 2) + (128 * j_5 + d_lane)] = dq_o_2; + pool_f32[(98304 - 32768 * th >> 2) + (1024 + 128 * j_5 + d_lane)] = dk_o_2; + } + float tot_2 = s_run_2[0]; + float dbp_2[8]; + tmem_ld_x8(&dbp_2[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_1); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1524[1]; - s_run_1524[0] = 0.0f; - int t_16 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1517 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1517 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1517 + 2944 + d_lane]; - float bt_t_16 = btv_1523[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1514); - float e_dg_16 = approx_exp2(g_t_16 - gc_1515); - float dq2_16 = _tmem_load_26[7] * e_dg_16; - float dk2_16 = _tmem_load_27[7] * e_dg_16; - dbp_1522[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_28[7]; - float dg_o_16 = __fmaf_rn(q_t_1_24, dq2_16, (dk2b_16 - _tmem_load_28[7]) * k_t_16) + dgf_16; - s_run_1524[0] = s_run_1524[0] + dg_o_16; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_16; - pool_f32[(98304 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_16; - int t_17 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1517 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1517 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1517 + 2816 + d_lane]; - float bt_t_17 = btv_1523[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1514); - float e_dg_17 = approx_exp2(g_t_17 - gc_1515); - float dq2_17 = _tmem_load_26[6] * e_dg_17; - float dk2_17 = _tmem_load_27[6] * e_dg_17; - dbp_1522[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[6]; - float dg_o_17 = __fmaf_rn(q_t_1_25, dq2_17, (dk2b_17 - _tmem_load_28[6]) * k_t_17) + dgf_17; - s_run_1524[0] = s_run_1524[0] + dg_o_17; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_17; - pool_f32[(98304 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_17; - int t_18 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1517 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1517 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1517 + 2688 + d_lane]; - float bt_t_18 = btv_1523[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1514); - float e_dg_18 = approx_exp2(g_t_18 - gc_1515); - float dq2_18 = _tmem_load_26[5] * e_dg_18; - float dk2_18 = _tmem_load_27[5] * e_dg_18; - dbp_1522[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[5]; - float dg_o_18 = __fmaf_rn(q_t_1_26, dq2_18, (dk2b_18 - _tmem_load_28[5]) * k_t_18) + dgf_18; - s_run_1524[0] = s_run_1524[0] + dg_o_18; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_18; - pool_f32[(98304 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_18; - int t_19 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1517 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1517 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1517 + 2560 + d_lane]; - float bt_t_19 = btv_1523[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1514); - float e_dg_19 = approx_exp2(g_t_19 - gc_1515); - float dq2_19 = _tmem_load_26[4] * e_dg_19; - float dk2_19 = _tmem_load_27[4] * e_dg_19; - dbp_1522[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[4]; - float dg_o_19 = __fmaf_rn(q_t_1_27, dq2_19, (dk2b_19 - _tmem_load_28[4]) * k_t_19) + dgf_19; - s_run_1524[0] = s_run_1524[0] + dg_o_19; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_19; - pool_f32[(98304 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_19; - int t_20 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1517 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1517 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1517 + 2432 + d_lane]; - float bt_t_20 = btv_1523[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1514); - float e_dg_20 = approx_exp2(g_t_20 - gc_1515); - float dq2_20 = _tmem_load_26[3] * e_dg_20; - float dk2_20 = _tmem_load_27[3] * e_dg_20; - dbp_1522[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[3]; - float dg_o_20 = __fmaf_rn(q_t_1_28, dq2_20, (dk2b_20 - _tmem_load_28[3]) * k_t_20) + dgf_20; - s_run_1524[0] = s_run_1524[0] + dg_o_20; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_20; - pool_f32[(98304 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_20; - int t_21 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1517 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1517 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1517 + 2304 + d_lane]; - float bt_t_21 = btv_1523[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1514); - float e_dg_21 = approx_exp2(g_t_21 - gc_1515); - float dq2_21 = _tmem_load_26[2] * e_dg_21; - float dk2_21 = _tmem_load_27[2] * e_dg_21; - dbp_1522[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[2]; - float dg_o_21 = __fmaf_rn(q_t_1_29, dq2_21, (dk2b_21 - _tmem_load_28[2]) * k_t_21) + dgf_21; - s_run_1524[0] = s_run_1524[0] + dg_o_21; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_21; - pool_f32[(98304 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_21; - int t_22 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1517 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1517 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1517 + 2176 + d_lane]; - float bt_t_22 = btv_1523[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1514); - float e_dg_22 = approx_exp2(g_t_22 - gc_1515); - float dq2_22 = _tmem_load_26[1] * e_dg_22; - float dk2_22 = _tmem_load_27[1] * e_dg_22; - dbp_1522[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[1]; - float dg_o_22 = __fmaf_rn(q_t_1_30, dq2_22, (dk2b_22 - _tmem_load_28[1]) * k_t_22) + dgf_22; - s_run_1524[0] = s_run_1524[0] + dg_o_22; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_22; - pool_f32[(98304 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_22; - int t_23 = 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1517 + d_lane]; - float dkf_23 = pool_f32[ew_1517 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1517 + 2048 + d_lane]; - float bt_t_23 = btv_1523[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1514); - float e_dg_23 = approx_exp2(g_t_23 - gc_1515); - float dq2_23 = _tmem_load_26[0] * e_dg_23; - float dk2_23 = _tmem_load_27[0] * e_dg_23; - dbp_1522[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[0]; - float dg_o_23 = __fmaf_rn(q_t_1_31, dq2_23, (dk2b_23 - _tmem_load_28[0]) * k_t_23) + dgf_23; - s_run_1524[0] = s_run_1524[0] + dg_o_23; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + d_lane] = dq_o_23; - pool_f32[(98304 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_23; - float tot_1693 = s_run_1524[0]; - int b4_1694 = lane_1 >> 4 & 1; - int b3_1695 = lane_1 >> 3 & 1; - int b2_1696 = lane_1 >> 2 & 1; - float v4_1697[4]; - float send_12 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_712 = __shfl_xor_sync(0xFFFFFFFF, send_12, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_712; - float send_13 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_713 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_713; - float send_14 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_714 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_714; - float send_15 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_715 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_715; - float v2_1702[2]; - float send_16 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_716 = __shfl_xor_sync(0xFFFFFFFF, send_16, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_716; - float send_17 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_717 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_717; - float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_718 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_718; - float _shfl_xor_sync_719 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_719; - float _shfl_xor_sync_720 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_720; - int ew_3 = 98304 - 32768 * th >> 2; + int b4_2 = lane_1 >> 4 & 1; + int b3_2 = lane_1 >> 3 & 1; + int b2_2 = lane_1 >> 2 & 1; + float v4_2[4]; + float send_8 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); + float _shfl_xor_sync_744 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_744; + float send_9 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); + float _shfl_xor_sync_745 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_745; + float send_10 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); + float _shfl_xor_sync_746 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_746; + float send_11 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); + float _shfl_xor_sync_747 = __shfl_xor_sync(0xFFFFFFFF, send_11, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_747; + float v2_17[2]; + float send2_4 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); + float _shfl_xor_sync_748 = __shfl_xor_sync(0xFFFFFFFF, send2_4, 8); + v2_17[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_748; + float send2_5 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); + float _shfl_xor_sync_749 = __shfl_xor_sync(0xFFFFFFFF, send2_5, 8); + v2_17[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_749; + float send1_1 = ((b2_2 == 0) ? v2_17[1] : v2_17[0]); + float _shfl_xor_sync_750 = __shfl_xor_sync(0xFFFFFFFF, send1_1, 4); + float v1_2 = ((b2_2 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_750; + float _shfl_xor_sync_751 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_751; + float _shfl_xor_sync_752 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_752; + int ew_6 = 98304 - 32768 * th >> 2; float nz_2 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); asm volatile("barrier.sync 1, 256;" ::: "memory"); #pragma unroll 1 for (int i_3 = 0; i_3 < 2; i_3++) { - int j_3 = qd + 4 * i_3; - bool live_3 = 8 * th + j_3 < clen; - int _min_721 = ((8 * th + j_3) < (clen - 1) ? (8 * th + j_3) : (clen - 1)); - long long row_3 = ext0 + (long long)_min_721; + int j_6 = qd + 4 * i_3; + bool live_3 = 8 * th + j_6 < clen; + int _min_753 = ((8 * th + j_6) < (clen - 1) ? (8 * th + j_6) : (clen - 1)); + long long row_3 = ext0 + (long long)_min_753; float gq_3[4]; float gk_3[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gq_3[0])), "=r"(*reinterpret_cast(&gq_3[(0) + 1])), "=r"(*reinterpret_cast(&gq_3[(0) + 2])), "=r"(*reinterpret_cast(&gq_3[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_3 + 128 * j_3 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_6 + 128 * j_6 + 4 * lane_1)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&gk_3[0])), "=r"(*reinterpret_cast(&gk_3[(0) + 1])), "=r"(*reinterpret_cast(&gk_3[(0) + 2])), "=r"(*reinterpret_cast(&gk_3[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(4 * (ew_3 + 1024 + 128 * j_3 + 4 * lane_1)))); + : "r"(pool_f32_addr + (unsigned int)(4 * (ew_6 + 1024 + 128 * j_6 + 4 * lane_1)))); float xq_3[4]; float xk_3[4]; float rq_3[1]; float rk_3[1]; { - uint2 _vld_722; - _vld_722 = *reinterpret_cast(q_raw + (row_3 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_722 = reinterpret_cast(&_vld_722); + uint2 _vld_754; + _vld_754 = *reinterpret_cast(q_raw + (row_3 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_754 = reinterpret_cast(&_vld_754); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xq_3[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_722[_pair]) << 16); - (&xq_3[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_722[_pair]) & 0xffff0000u); + (&xq_3[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_754[_pair]) << 16); + (&xq_3[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_754[_pair]) & 0xffff0000u); } } { rq_3[0] = *reinterpret_cast(qk_rstd + ((0 + row_3) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_723; - _vld_723 = *reinterpret_cast(k_raw + (row_3 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_723 = reinterpret_cast(&_vld_723); + uint2 _vld_755; + _vld_755 = *reinterpret_cast(k_raw + (row_3 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_755 = reinterpret_cast(&_vld_755); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xk_3[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_723[_pair]) << 16); - (&xk_3[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_723[_pair]) & 0xffff0000u); + (&xk_3[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_755[_pair]) << 16); + (&xk_3[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_755[_pair]) & 0xffff0000u); } } { @@ -6177,106 +5759,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_3[4]; float p_3[4]; y_3[0] = mul_f32_rn_noftz(xq_3[0], rq_3[0]); - float _fma_724; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_724) : "f"(gq_3[0]), "f"(y_3[0]), "f"(nz_2)); - p_3[0] = _fma_724; + float _fma_756; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_756) : "f"(gq_3[0]), "f"(y_3[0]), "f"(nz_2)); + p_3[0] = _fma_756; y_3[1] = mul_f32_rn_noftz(xq_3[1], rq_3[0]); - float _fma_725; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_725) : "f"(gq_3[1]), "f"(y_3[1]), "f"(nz_2)); - p_3[1] = _fma_725; + float _fma_757; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_757) : "f"(gq_3[1]), "f"(y_3[1]), "f"(nz_2)); + p_3[1] = _fma_757; y_3[2] = mul_f32_rn_noftz(xq_3[2], rq_3[0]); - float _fma_726; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_726) : "f"(gq_3[2]), "f"(y_3[2]), "f"(nz_2)); - p_3[2] = _fma_726; + float _fma_758; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_758) : "f"(gq_3[2]), "f"(y_3[2]), "f"(nz_2)); + p_3[2] = _fma_758; y_3[3] = mul_f32_rn_noftz(xq_3[3], rq_3[0]); - float _fma_727; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_727) : "f"(gq_3[3]), "f"(y_3[3]), "f"(nz_2)); - p_3[3] = _fma_727; - float _shfl_xor_sync_729 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 4); - float _add_728; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_728) : "f"(p_3[0]), "f"(_shfl_xor_sync_729)); - p_3[0] = _add_728; - float _shfl_xor_sync_731 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 4); - float _add_730; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_730) : "f"(p_3[1]), "f"(_shfl_xor_sync_731)); - p_3[1] = _add_730; - float _shfl_xor_sync_733 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 4); - float _add_732; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_732) : "f"(p_3[2]), "f"(_shfl_xor_sync_733)); - p_3[2] = _add_732; - float _shfl_xor_sync_735 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 4); - float _add_734; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_734) : "f"(p_3[3]), "f"(_shfl_xor_sync_735)); - p_3[3] = _add_734; - float _shfl_xor_sync_737 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 2); - float _add_736; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_736) : "f"(p_3[0]), "f"(_shfl_xor_sync_737)); - p_3[0] = _add_736; - float _shfl_xor_sync_739 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 2); - float _add_738; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_738) : "f"(p_3[1]), "f"(_shfl_xor_sync_739)); - p_3[1] = _add_738; - float _shfl_xor_sync_741 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 2); - float _add_740; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_740) : "f"(p_3[2]), "f"(_shfl_xor_sync_741)); - p_3[2] = _add_740; - float _shfl_xor_sync_743 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 2); - float _add_742; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_742) : "f"(p_3[3]), "f"(_shfl_xor_sync_743)); - p_3[3] = _add_742; - float _shfl_xor_sync_745 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 1); - float _add_744; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_744) : "f"(p_3[0]), "f"(_shfl_xor_sync_745)); - p_3[0] = _add_744; - float _shfl_xor_sync_747 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 1); - float _add_746; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_746) : "f"(p_3[1]), "f"(_shfl_xor_sync_747)); - p_3[1] = _add_746; - float _shfl_xor_sync_749 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 1); - float _add_748; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_748) : "f"(p_3[2]), "f"(_shfl_xor_sync_749)); - p_3[2] = _add_748; - float _shfl_xor_sync_751 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 1); - float _add_750; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_750) : "f"(p_3[3]), "f"(_shfl_xor_sync_751)); - p_3[3] = _add_750; - float _add_752; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_752) : "f"(p_3[0]), "f"(p_3[2])); - float _add_753; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_753) : "f"(p_3[1]), "f"(p_3[3])); + float _fma_759; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_759) : "f"(gq_3[3]), "f"(y_3[3]), "f"(nz_2)); + p_3[3] = _fma_759; + float _shfl_xor_sync_761 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 4); + float _add_760; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_760) : "f"(p_3[0]), "f"(_shfl_xor_sync_761)); + p_3[0] = _add_760; + float _shfl_xor_sync_763 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 4); + float _add_762; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_762) : "f"(p_3[1]), "f"(_shfl_xor_sync_763)); + p_3[1] = _add_762; + float _shfl_xor_sync_765 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 4); + float _add_764; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_764) : "f"(p_3[2]), "f"(_shfl_xor_sync_765)); + p_3[2] = _add_764; + float _shfl_xor_sync_767 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 4); + float _add_766; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_766) : "f"(p_3[3]), "f"(_shfl_xor_sync_767)); + p_3[3] = _add_766; + float _shfl_xor_sync_769 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 2); + float _add_768; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_768) : "f"(p_3[0]), "f"(_shfl_xor_sync_769)); + p_3[0] = _add_768; + float _shfl_xor_sync_771 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 2); + float _add_770; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_770) : "f"(p_3[1]), "f"(_shfl_xor_sync_771)); + p_3[1] = _add_770; + float _shfl_xor_sync_773 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 2); + float _add_772; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_772) : "f"(p_3[2]), "f"(_shfl_xor_sync_773)); + p_3[2] = _add_772; + float _shfl_xor_sync_775 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 2); + float _add_774; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_774) : "f"(p_3[3]), "f"(_shfl_xor_sync_775)); + p_3[3] = _add_774; + float _shfl_xor_sync_777 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 1); + float _add_776; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_776) : "f"(p_3[0]), "f"(_shfl_xor_sync_777)); + p_3[0] = _add_776; + float _shfl_xor_sync_779 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 1); + float _add_778; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_778) : "f"(p_3[1]), "f"(_shfl_xor_sync_779)); + p_3[1] = _add_778; + float _shfl_xor_sync_781 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 1); + float _add_780; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_780) : "f"(p_3[2]), "f"(_shfl_xor_sync_781)); + p_3[2] = _add_780; + float _shfl_xor_sync_783 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 1); + float _add_782; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_782) : "f"(p_3[3]), "f"(_shfl_xor_sync_783)); + p_3[3] = _add_782; + float _add_784; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_784) : "f"(p_3[0]), "f"(p_3[2])); + float _add_785; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_785) : "f"(p_3[1]), "f"(p_3[3])); float sq_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3) : "f"(_add_752), "f"(_add_753)); - float _shfl_sync_756 = __shfl_sync(0xFFFFFFFF, sq_3, 0); - float _shfl_sync_757 = __shfl_sync(0xFFFFFFFF, sq_3, 8); - float _add_755; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_755) : "f"(_shfl_sync_756), "f"(_shfl_sync_757)); - float _shfl_sync_758 = __shfl_sync(0xFFFFFFFF, sq_3, 16); - float _add_754; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_754) : "f"(_add_755), "f"(_shfl_sync_758)); - float _shfl_sync_759 = __shfl_sync(0xFFFFFFFF, sq_3, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3) : "f"(_add_784), "f"(_add_785)); + float _shfl_sync_788 = __shfl_sync(0xFFFFFFFF, sq_3, 0); + float _shfl_sync_789 = __shfl_sync(0xFFFFFFFF, sq_3, 8); + float _add_787; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_787) : "f"(_shfl_sync_788), "f"(_shfl_sync_789)); + float _shfl_sync_790 = __shfl_sync(0xFFFFFFFF, sq_3, 16); + float _add_786; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_786) : "f"(_add_787), "f"(_shfl_sync_790)); + float _shfl_sync_791 = __shfl_sync(0xFFFFFFFF, sq_3, 24); float dot_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3) : "f"(_add_754), "f"(_shfl_sync_759)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3) : "f"(_add_786), "f"(_shfl_sync_791)); float o_3[4]; - float _fma_761; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_761) : "f"(y_3[0]), "f"(dot_3), "f"(nz_2)); - float _sub_760; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_760) : "f"(gq_3[0]), "f"(_fma_761)); - o_3[0] = mul_f32_rn_noftz(rq_3[0], _sub_760); - float _fma_763; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_763) : "f"(y_3[1]), "f"(dot_3), "f"(nz_2)); - float _sub_762; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_762) : "f"(gq_3[1]), "f"(_fma_763)); - o_3[1] = mul_f32_rn_noftz(rq_3[0], _sub_762); - float _fma_765; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_765) : "f"(y_3[2]), "f"(dot_3), "f"(nz_2)); - float _sub_764; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_764) : "f"(gq_3[2]), "f"(_fma_765)); - o_3[2] = mul_f32_rn_noftz(rq_3[0], _sub_764); - float _fma_767; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_767) : "f"(y_3[3]), "f"(dot_3), "f"(nz_2)); - float _sub_766; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_766) : "f"(gq_3[3]), "f"(_fma_767)); - o_3[3] = mul_f32_rn_noftz(rq_3[0], _sub_766); + float _fma_793; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_793) : "f"(y_3[0]), "f"(dot_3), "f"(nz_2)); + float _sub_792; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_792) : "f"(gq_3[0]), "f"(_fma_793)); + o_3[0] = mul_f32_rn_noftz(rq_3[0], _sub_792); + float _fma_795; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_795) : "f"(y_3[1]), "f"(dot_3), "f"(nz_2)); + float _sub_794; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_794) : "f"(gq_3[1]), "f"(_fma_795)); + o_3[1] = mul_f32_rn_noftz(rq_3[0], _sub_794); + float _fma_797; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_797) : "f"(y_3[2]), "f"(dot_3), "f"(nz_2)); + float _sub_796; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_796) : "f"(gq_3[2]), "f"(_fma_797)); + o_3[2] = mul_f32_rn_noftz(rq_3[0], _sub_796); + float _fma_799; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_799) : "f"(y_3[3]), "f"(dot_3), "f"(nz_2)); + float _sub_798; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_798) : "f"(gq_3[3]), "f"(_fma_799)); + o_3[3] = mul_f32_rn_noftz(rq_3[0], _sub_798); unsigned int ob_3[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -6287,106 +5869,106 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float y_3_1[4]; float p_3_1[4]; y_3_1[0] = mul_f32_rn_noftz(xk_3[0], rk_3[0]); - float _fma_768; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_768) : "f"(gk_3[0]), "f"(y_3_1[0]), "f"(nz_2)); - p_3_1[0] = _fma_768; + float _fma_800; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_800) : "f"(gk_3[0]), "f"(y_3_1[0]), "f"(nz_2)); + p_3_1[0] = _fma_800; y_3_1[1] = mul_f32_rn_noftz(xk_3[1], rk_3[0]); - float _fma_769; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_769) : "f"(gk_3[1]), "f"(y_3_1[1]), "f"(nz_2)); - p_3_1[1] = _fma_769; + float _fma_801; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_801) : "f"(gk_3[1]), "f"(y_3_1[1]), "f"(nz_2)); + p_3_1[1] = _fma_801; y_3_1[2] = mul_f32_rn_noftz(xk_3[2], rk_3[0]); - float _fma_770; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_770) : "f"(gk_3[2]), "f"(y_3_1[2]), "f"(nz_2)); - p_3_1[2] = _fma_770; + float _fma_802; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_802) : "f"(gk_3[2]), "f"(y_3_1[2]), "f"(nz_2)); + p_3_1[2] = _fma_802; y_3_1[3] = mul_f32_rn_noftz(xk_3[3], rk_3[0]); - float _fma_771; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_771) : "f"(gk_3[3]), "f"(y_3_1[3]), "f"(nz_2)); - p_3_1[3] = _fma_771; - float _shfl_xor_sync_773 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 4); - float _add_772; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_772) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_773)); - p_3_1[0] = _add_772; - float _shfl_xor_sync_775 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 4); - float _add_774; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_774) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_775)); - p_3_1[1] = _add_774; - float _shfl_xor_sync_777 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 4); - float _add_776; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_776) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_777)); - p_3_1[2] = _add_776; - float _shfl_xor_sync_779 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 4); - float _add_778; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_778) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_779)); - p_3_1[3] = _add_778; - float _shfl_xor_sync_781 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 2); - float _add_780; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_780) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_781)); - p_3_1[0] = _add_780; - float _shfl_xor_sync_783 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 2); - float _add_782; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_782) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_783)); - p_3_1[1] = _add_782; - float _shfl_xor_sync_785 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 2); - float _add_784; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_784) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_785)); - p_3_1[2] = _add_784; - float _shfl_xor_sync_787 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 2); - float _add_786; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_786) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_787)); - p_3_1[3] = _add_786; - float _shfl_xor_sync_789 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 1); - float _add_788; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_788) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_789)); - p_3_1[0] = _add_788; - float _shfl_xor_sync_791 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 1); - float _add_790; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_790) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_791)); - p_3_1[1] = _add_790; - float _shfl_xor_sync_793 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 1); - float _add_792; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_792) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_793)); - p_3_1[2] = _add_792; - float _shfl_xor_sync_795 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 1); - float _add_794; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_794) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_795)); - p_3_1[3] = _add_794; - float _add_796; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_796) : "f"(p_3_1[0]), "f"(p_3_1[2])); - float _add_797; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_797) : "f"(p_3_1[1]), "f"(p_3_1[3])); + float _fma_803; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_803) : "f"(gk_3[3]), "f"(y_3_1[3]), "f"(nz_2)); + p_3_1[3] = _fma_803; + float _shfl_xor_sync_805 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 4); + float _add_804; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_804) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_805)); + p_3_1[0] = _add_804; + float _shfl_xor_sync_807 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 4); + float _add_806; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_806) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_807)); + p_3_1[1] = _add_806; + float _shfl_xor_sync_809 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 4); + float _add_808; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_808) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_809)); + p_3_1[2] = _add_808; + float _shfl_xor_sync_811 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 4); + float _add_810; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_810) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_811)); + p_3_1[3] = _add_810; + float _shfl_xor_sync_813 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 2); + float _add_812; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_812) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_813)); + p_3_1[0] = _add_812; + float _shfl_xor_sync_815 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 2); + float _add_814; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_814) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_815)); + p_3_1[1] = _add_814; + float _shfl_xor_sync_817 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 2); + float _add_816; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_816) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_817)); + p_3_1[2] = _add_816; + float _shfl_xor_sync_819 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 2); + float _add_818; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_818) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_819)); + p_3_1[3] = _add_818; + float _shfl_xor_sync_821 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 1); + float _add_820; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_820) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_821)); + p_3_1[0] = _add_820; + float _shfl_xor_sync_823 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 1); + float _add_822; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_822) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_823)); + p_3_1[1] = _add_822; + float _shfl_xor_sync_825 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 1); + float _add_824; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_824) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_825)); + p_3_1[2] = _add_824; + float _shfl_xor_sync_827 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 1); + float _add_826; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_826) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_827)); + p_3_1[3] = _add_826; + float _add_828; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_828) : "f"(p_3_1[0]), "f"(p_3_1[2])); + float _add_829; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_829) : "f"(p_3_1[1]), "f"(p_3_1[3])); float sq_3_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3_1) : "f"(_add_796), "f"(_add_797)); - float _shfl_sync_800 = __shfl_sync(0xFFFFFFFF, sq_3_1, 0); - float _shfl_sync_801 = __shfl_sync(0xFFFFFFFF, sq_3_1, 8); - float _add_799; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_799) : "f"(_shfl_sync_800), "f"(_shfl_sync_801)); - float _shfl_sync_802 = __shfl_sync(0xFFFFFFFF, sq_3_1, 16); - float _add_798; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_798) : "f"(_add_799), "f"(_shfl_sync_802)); - float _shfl_sync_803 = __shfl_sync(0xFFFFFFFF, sq_3_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3_1) : "f"(_add_828), "f"(_add_829)); + float _shfl_sync_832 = __shfl_sync(0xFFFFFFFF, sq_3_1, 0); + float _shfl_sync_833 = __shfl_sync(0xFFFFFFFF, sq_3_1, 8); + float _add_831; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_831) : "f"(_shfl_sync_832), "f"(_shfl_sync_833)); + float _shfl_sync_834 = __shfl_sync(0xFFFFFFFF, sq_3_1, 16); + float _add_830; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_830) : "f"(_add_831), "f"(_shfl_sync_834)); + float _shfl_sync_835 = __shfl_sync(0xFFFFFFFF, sq_3_1, 24); float dot_3_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3_1) : "f"(_add_798), "f"(_shfl_sync_803)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3_1) : "f"(_add_830), "f"(_shfl_sync_835)); float o_3_1[4]; - float _fma_805; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_805) : "f"(y_3_1[0]), "f"(dot_3_1), "f"(nz_2)); - float _sub_804; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_804) : "f"(gk_3[0]), "f"(_fma_805)); - o_3_1[0] = mul_f32_rn_noftz(rk_3[0], _sub_804); - float _fma_807; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_807) : "f"(y_3_1[1]), "f"(dot_3_1), "f"(nz_2)); - float _sub_806; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_806) : "f"(gk_3[1]), "f"(_fma_807)); - o_3_1[1] = mul_f32_rn_noftz(rk_3[0], _sub_806); - float _fma_809; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_809) : "f"(y_3_1[2]), "f"(dot_3_1), "f"(nz_2)); - float _sub_808; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_808) : "f"(gk_3[2]), "f"(_fma_809)); - o_3_1[2] = mul_f32_rn_noftz(rk_3[0], _sub_808); - float _fma_811; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_811) : "f"(y_3_1[3]), "f"(dot_3_1), "f"(nz_2)); - float _sub_810; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_810) : "f"(gk_3[3]), "f"(_fma_811)); - o_3_1[3] = mul_f32_rn_noftz(rk_3[0], _sub_810); + float _fma_837; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_837) : "f"(y_3_1[0]), "f"(dot_3_1), "f"(nz_2)); + float _sub_836; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_836) : "f"(gk_3[0]), "f"(_fma_837)); + o_3_1[0] = mul_f32_rn_noftz(rk_3[0], _sub_836); + float _fma_839; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_839) : "f"(y_3_1[1]), "f"(dot_3_1), "f"(nz_2)); + float _sub_838; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_838) : "f"(gk_3[1]), "f"(_fma_839)); + o_3_1[1] = mul_f32_rn_noftz(rk_3[0], _sub_838); + float _fma_841; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_841) : "f"(y_3_1[2]), "f"(dot_3_1), "f"(nz_2)); + float _sub_840; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_840) : "f"(gk_3[2]), "f"(_fma_841)); + o_3_1[2] = mul_f32_rn_noftz(rk_3[0], _sub_840); + float _fma_843; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_843) : "f"(y_3_1[3]), "f"(dot_3_1), "f"(nz_2)); + float _sub_842; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_842) : "f"(gk_3[3]), "f"(_fma_843)); + o_3_1[3] = mul_f32_rn_noftz(rk_3[0], _sub_842); unsigned int ob_3_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -6399,8 +5981,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_812 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_812), "f"(tot_1693) : "memory"); + uint32_t _addr_844 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_844), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -6418,8 +6000,8 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other_1[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other_1[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi_2 = ((th == 1) ? tot_1693 : t_other_1[0]); - float t_lo_2 = ((th == 1) ? t_other_1[0] : tot_1693); + float t_hi_2 = ((th == 1) ? tot_2 : t_other_1[0]); + float t_lo_2 = ((th == 1) ? t_other_1[0] : tot_2); float offset_2 = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi_2); float sfx_2[8]; tmem_ld_x8(&sfx_2[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); @@ -6432,132 +6014,132 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_813 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_845 = make_float2(gacc[4], gacc[4]); float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_813), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_816; - asm("abs.f32 %0, %1;" : "=f"(_abs_816) : "f"(rp_z_11.x)); - float _neg_815; - asm("neg.f32 %0, %1;" : "=f"(_neg_815) : "f"(_abs_816)); - float _exp_814 = __nv_expf(_neg_815); - float _abs_819; - asm("abs.f32 %0, %1;" : "=f"(_abs_819) : "f"(rp_z_11.y)); - float _neg_818; - asm("neg.f32 %0, %1;" : "=f"(_neg_818) : "f"(_abs_819)); - float _exp_817 = __nv_expf(_neg_818); - float2 rp_e_11 = make_float2(_exp_814, _exp_817); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_845), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_848; + asm("abs.f32 %0, %1;" : "=f"(_abs_848) : "f"(rp_z_11.x)); + float _neg_847; + asm("neg.f32 %0, %1;" : "=f"(_neg_847) : "f"(_abs_848)); + float _exp_846 = __nv_expf(_neg_847); + float _abs_851; + asm("abs.f32 %0, %1;" : "=f"(_abs_851) : "f"(rp_z_11.y)); + float _neg_850; + asm("neg.f32 %0, %1;" : "=f"(_neg_850) : "f"(_abs_851)); + float _exp_849 = __nv_expf(_neg_850); + float2 rp_e_11 = make_float2(_exp_846, _exp_849); float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_820 = approx_rcp(rp_den_11.x); - float _rcp_821 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_822 = make_float2(_rcp_820, _rcp_821); + float _rcp_852 = approx_rcp(rp_den_11.x); + float _rcp_853 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_854 = make_float2(_rcp_852, _rcp_853); float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_822)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_854)); float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_823 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_855 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_823)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_855)); u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_824 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_856 = make_float2(gacc[4], gacc[4]); float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_824), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_827; - asm("abs.f32 %0, %1;" : "=f"(_abs_827) : "f"(rp_z_12.x)); - float _neg_826; - asm("neg.f32 %0, %1;" : "=f"(_neg_826) : "f"(_abs_827)); - float _exp_825 = __nv_expf(_neg_826); - float _abs_830; - asm("abs.f32 %0, %1;" : "=f"(_abs_830) : "f"(rp_z_12.y)); - float _neg_829; - asm("neg.f32 %0, %1;" : "=f"(_neg_829) : "f"(_abs_830)); - float _exp_828 = __nv_expf(_neg_829); - float2 rp_e_12 = make_float2(_exp_825, _exp_828); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_856), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_859; + asm("abs.f32 %0, %1;" : "=f"(_abs_859) : "f"(rp_z_12.x)); + float _neg_858; + asm("neg.f32 %0, %1;" : "=f"(_neg_858) : "f"(_abs_859)); + float _exp_857 = __nv_expf(_neg_858); + float _abs_862; + asm("abs.f32 %0, %1;" : "=f"(_abs_862) : "f"(rp_z_12.y)); + float _neg_861; + asm("neg.f32 %0, %1;" : "=f"(_neg_861) : "f"(_abs_862)); + float _exp_860 = __nv_expf(_neg_861); + float2 rp_e_12 = make_float2(_exp_857, _exp_860); float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_831 = approx_rcp(rp_den_12.x); - float _rcp_832 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_833 = make_float2(_rcp_831, _rcp_832); + float _rcp_863 = approx_rcp(rp_den_12.x); + float _rcp_864 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_865 = make_float2(_rcp_863, _rcp_864); float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_833)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_865)); float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_834 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_866 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_834)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_866)); u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_835 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_867 = make_float2(gacc[4], gacc[4]); float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_835), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_838; - asm("abs.f32 %0, %1;" : "=f"(_abs_838) : "f"(rp_z_13.x)); - float _neg_837; - asm("neg.f32 %0, %1;" : "=f"(_neg_837) : "f"(_abs_838)); - float _exp_836 = __nv_expf(_neg_837); - float _abs_841; - asm("abs.f32 %0, %1;" : "=f"(_abs_841) : "f"(rp_z_13.y)); - float _neg_840; - asm("neg.f32 %0, %1;" : "=f"(_neg_840) : "f"(_abs_841)); - float _exp_839 = __nv_expf(_neg_840); - float2 rp_e_13 = make_float2(_exp_836, _exp_839); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_867), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_870; + asm("abs.f32 %0, %1;" : "=f"(_abs_870) : "f"(rp_z_13.x)); + float _neg_869; + asm("neg.f32 %0, %1;" : "=f"(_neg_869) : "f"(_abs_870)); + float _exp_868 = __nv_expf(_neg_869); + float _abs_873; + asm("abs.f32 %0, %1;" : "=f"(_abs_873) : "f"(rp_z_13.y)); + float _neg_872; + asm("neg.f32 %0, %1;" : "=f"(_neg_872) : "f"(_abs_873)); + float _exp_871 = __nv_expf(_neg_872); + float2 rp_e_13 = make_float2(_exp_868, _exp_871); float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_842 = approx_rcp(rp_den_13.x); - float _rcp_843 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_844 = make_float2(_rcp_842, _rcp_843); + float _rcp_874 = approx_rcp(rp_den_13.x); + float _rcp_875 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_876 = make_float2(_rcp_874, _rcp_875); float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_844)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_876)); float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_845 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_877 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_845)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_877)); u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_846 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_878 = make_float2(gacc[4], gacc[4]); float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_846), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_849; - asm("abs.f32 %0, %1;" : "=f"(_abs_849) : "f"(rp_z_14.x)); - float _neg_848; - asm("neg.f32 %0, %1;" : "=f"(_neg_848) : "f"(_abs_849)); - float _exp_847 = __nv_expf(_neg_848); - float _abs_852; - asm("abs.f32 %0, %1;" : "=f"(_abs_852) : "f"(rp_z_14.y)); - float _neg_851; - asm("neg.f32 %0, %1;" : "=f"(_neg_851) : "f"(_abs_852)); - float _exp_850 = __nv_expf(_neg_851); - float2 rp_e_14 = make_float2(_exp_847, _exp_850); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_878), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_881; + asm("abs.f32 %0, %1;" : "=f"(_abs_881) : "f"(rp_z_14.x)); + float _neg_880; + asm("neg.f32 %0, %1;" : "=f"(_neg_880) : "f"(_abs_881)); + float _exp_879 = __nv_expf(_neg_880); + float _abs_884; + asm("abs.f32 %0, %1;" : "=f"(_abs_884) : "f"(rp_z_14.y)); + float _neg_883; + asm("neg.f32 %0, %1;" : "=f"(_neg_883) : "f"(_abs_884)); + float _exp_882 = __nv_expf(_neg_883); + float2 rp_e_14 = make_float2(_exp_879, _exp_882); float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_853 = approx_rcp(rp_den_14.x); - float _rcp_854 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_855 = make_float2(_rcp_853, _rcp_854); + float _rcp_885 = approx_rcp(rp_den_14.x); + float _rcp_886 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_887 = make_float2(_rcp_885, _rcp_886); float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_855)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_887)); float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_856 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_888 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_856)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_888)); u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -6572,54 +6154,54 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_857; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_857) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_857; + uint32_t _prmt_889; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_889) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_889; int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_858 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_858)); - *(reinterpret_cast(_gst_858) + (0)) = pq_12[0]; + auto _gst_890 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_890)); + *(reinterpret_cast(_gst_890) + (0)) = pq_12[0]; } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_859; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_859) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_859; + uint32_t _prmt_891; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_891) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_891; int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_860 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_860)); - *(reinterpret_cast(_gst_860) + (0)) = pq_13[0]; + auto _gst_892 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_892)); + *(reinterpret_cast(_gst_892) + (0)) = pq_13[0]; } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_861; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_861) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_861; + uint32_t _prmt_893; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_893) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_893; int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_862 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_862)); - *(reinterpret_cast(_gst_862) + (0)) = pq_14[0]; + auto _gst_894 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_894)); + *(reinterpret_cast(_gst_894) + (0)) = pq_14[0]; } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_863; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_863) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_863; + uint32_t _prmt_895; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_895) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_895; int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_864 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_864)); - *(reinterpret_cast(_gst_864) + (0)) = pq_15[0]; + auto _gst_896 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_896)); + *(reinterpret_cast(_gst_896) + (0)) = pq_15[0]; } } int t_a_24 = 8 * th; @@ -6684,47 +6266,47 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_865; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_865) : "f"(s_a_2), "f"(nm_t_5)); + float _sub_897; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_897) : "f"(s_a_2), "f"(nm_t_5)); float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_865), "f"(pa_2)); - float _sub_866; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_866) : "f"(pa_2), "f"(nm_t_5)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_897), "f"(pa_2)); + float _sub_898; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_898) : "f"(pa_2), "f"(nm_t_5)); float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_866), "f"(s_a_2)); - float _abs_868; - asm("abs.f32 %0, %1;" : "=f"(_abs_868) : "f"(s_a_2)); - float _abs_869; - asm("abs.f32 %0, %1;" : "=f"(_abs_869) : "f"(pa_2)); - uint32_t _setp_867_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_867_bits) : "f"(_abs_868), "f"(_abs_869)); - bool _setp_867 = _setp_867_bits != 0; - float nm_err_5 = ((_setp_867) ? nm_es_5 : nm_ex_5); - float _add_870; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_870) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_870; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_898), "f"(s_a_2)); + float _abs_900; + asm("abs.f32 %0, %1;" : "=f"(_abs_900) : "f"(s_a_2)); + float _abs_901; + asm("abs.f32 %0, %1;" : "=f"(_abs_901) : "f"(pa_2)); + uint32_t _setp_899_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_899_bits) : "f"(_abs_900), "f"(_abs_901)); + bool _setp_899 = _setp_899_bits != 0; + float nm_err_5 = ((_setp_899) ? nm_es_5 : nm_ex_5); + float _add_902; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_902) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_902; s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_871; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_871) : "f"(s_b_2), "f"(nm_t_6)); + float _sub_903; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_903) : "f"(s_b_2), "f"(nm_t_6)); float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_871), "f"(pb_2)); - float _sub_872; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_872) : "f"(pb_2), "f"(nm_t_6)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_903), "f"(pb_2)); + float _sub_904; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_904) : "f"(pb_2), "f"(nm_t_6)); float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_872), "f"(s_b_2)); - float _abs_874; - asm("abs.f32 %0, %1;" : "=f"(_abs_874) : "f"(s_b_2)); - float _abs_875; - asm("abs.f32 %0, %1;" : "=f"(_abs_875) : "f"(pb_2)); - uint32_t _setp_873_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_873_bits) : "f"(_abs_874), "f"(_abs_875)); - bool _setp_873 = _setp_873_bits != 0; - float nm_err_6 = ((_setp_873) ? nm_es_6 : nm_ex_6); - float _add_876; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_876) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_876; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_904), "f"(s_b_2)); + float _abs_906; + asm("abs.f32 %0, %1;" : "=f"(_abs_906) : "f"(s_b_2)); + float _abs_907; + asm("abs.f32 %0, %1;" : "=f"(_abs_907) : "f"(pb_2)); + uint32_t _setp_905_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_905_bits) : "f"(_abs_906), "f"(_abs_907)); + bool _setp_905 = _setp_905_bits != 0; + float nm_err_6 = ((_setp_905) ? nm_es_6 : nm_ex_6); + float _add_908; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_908) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_908; s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -6763,60 +6345,60 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_878; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_878) : "f"(ts_s), "f"(ts_bb)); - float _sub_877; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_877) : "f"(s_a_3), "f"(_sub_878)); - float _sub_879; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_879) : "f"(other[0]), "f"(ts_bb)); + float _sub_910; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_910) : "f"(ts_s), "f"(ts_bb)); + float _sub_909; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_909) : "f"(s_a_3), "f"(_sub_910)); + float _sub_911; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_911) : "f"(other[0]), "f"(ts_bb)); float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_877), "f"(_sub_879)); - float _add_880; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_880) : "f"(c_a_3), "f"(other[1])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_909), "f"(_sub_911)); + float _add_912; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_912) : "f"(c_a_3), "f"(other[1])); float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_880), "f"(ts_e)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_912), "f"(ts_e)); float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_882; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_882) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_881; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_881) : "f"(ts_s), "f"(_sub_882)); - float _sub_883; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_883) : "f"(pm_c), "f"(ts_bb_0)); + float _sub_914; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_914) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_913; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_913) : "f"(ts_s), "f"(_sub_914)); + float _sub_915; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_915) : "f"(pm_c), "f"(ts_bb_0)); float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_881), "f"(_sub_883)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_913), "f"(_sub_915)); s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_885; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_885) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_884; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_884) : "f"(s_b_3), "f"(_sub_885)); - float _sub_886; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_886) : "f"(other[2]), "f"(ts_bb_1)); + float _sub_917; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_917) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_916; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_916) : "f"(s_b_3), "f"(_sub_917)); + float _sub_918; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_918) : "f"(other[2]), "f"(ts_bb_1)); float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_884), "f"(_sub_886)); - float _add_887; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_887) : "f"(c_b_3), "f"(other[3])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_916), "f"(_sub_918)); + float _add_919; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_919) : "f"(c_b_3), "f"(other[3])); float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_887), "f"(ts_e_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_919), "f"(ts_e_1)); float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_889; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_889) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_888; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_888) : "f"(ts_s_1), "f"(_sub_889)); - float _sub_890; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_890) : "f"(pm_c_0), "f"(ts_bb_2)); + float _sub_921; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_921) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_920; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_920) : "f"(ts_s_1), "f"(_sub_921)); + float _sub_922; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_922) : "f"(pm_c_0), "f"(ts_bb_2)); float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_888), "f"(_sub_890)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_920), "f"(_sub_922)); s_b_3 = ts_s_2; c_b_3 = ts_e_2; gacc[0] = s_a_3; @@ -6834,10 +6416,10 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM } if ((lane_1 & 3) == 0) { int e_star = lane_1 >> 2 & 7; - pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_1706; - pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1484; + pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_2; + pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1; pool_f32[28160 + qd * 64 + 32 + 8 * th + e_star] = v1; - pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_1241; + pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_0; } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync 1, 256;" ::: "memory"); @@ -6927,41 +6509,41 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + for (int r_284 = 0; r_284 < 16; r_284++) { + int t_3 = r_284; + if (t_3 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -7033,41 +6615,41 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_24 = 16 + r_348; - if (t_24 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + for (int r_285 = 0; r_285 < 16; r_285++) { + int t_4 = 16 + r_285; + if (t_4 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -7183,41 +6765,41 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_25 = 32 + r_349; - if (t_25 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + for (int r_286 = 0; r_286 < 16; r_286++) { + int t_5 = 32 + r_286; + if (t_5 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -7365,41 +6947,41 @@ kernel_cake_kda_det_train_6308ef19a5bf0691fcfe(const __grid_constant__ CUtensorM z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_26 = 48 + r_350; - if (t_26 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + for (int r_287 = 0; r_287 < 16; r_287++) { + int t_6 = 48 + r_287; + if (t_6 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_binding.cu similarity index 98% rename from flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu rename to flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_binding.cu index 9790bcabdfd..73779f19232 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_binding.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_kda_det_train_328afe43951979e798d1, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_kda_det_train_a135a53a639f33f921ab, for the device source that defines it. Do not edit. #include #include #include @@ -23,13 +23,13 @@ #include -extern "C" __global__ void kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride); +extern "C" __global__ void kernel_cake_kda_det_train_a135a53a639f33f921ab(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride); -namespace cake_binding_kernel_cake_kda_det_train_328afe43951979e798d1 { +namespace cake_binding_kernel_cake_kda_det_train_a135a53a639f33f921ab { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_kda_det_train_328afe43951979e798d1"; +constexpr const char* KERNEL_NAME = "kernel_cake_kda_det_train_a135a53a639f33f921ab"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -487,7 +487,7 @@ void Run(TensorView a_dAqk, TensorView a_dAkk, TensorView a_gk, TensorView a_k_e const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { CheckCuda(cudaSetDevice(dev.device_id), "cudaSetDevice"); - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_kda_det_train_328afe43951979e798d1), cudaFuncAttributeMaxDynamicSharedMemorySize, 115200), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_kda_det_train_a135a53a639f33f921ab), cudaFuncAttributeMaxDynamicSharedMemorySize, 115200), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -527,11 +527,11 @@ void Run(TensorView a_dAqk, TensorView a_dAkk, TensorView a_gk, TensorView a_k_e config.blockDim = dim3(288, 1, 1); config.dynamicSmemBytes = 115200; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_kda_det_train_328afe43951979e798d1), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_kda_det_train_a135a53a639f33f921ab), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_kda_det_train_328afe43951979e798d1 +} // namespace cake_binding_kernel_cake_kda_det_train_a135a53a639f33f921ab -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_kda_det_train_328afe43951979e798d1::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_kda_det_train_a135a53a639f33f921ab::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_kernel.cu similarity index 90% rename from flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_kernel.cu rename to flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_kernel.cu index 05f7949c75c..5744991ab5f 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_a135a53a639f33f921ab_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 add_f32x2_noftz(float2 a, float2 b) { float2 r; asm("add.rn.f32x2 %0, %1, %2;" @@ -207,10 +214,26 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void -kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride) +kernel_cake_kda_det_train_a135a53a639f33f921ab(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -2275,12 +2298,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_3 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * e2_3); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_3, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_3 : __fmaf_rn(dl, _rcp_170, e2_3)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2304,12 +2333,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_4 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * e2_4); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_4, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_4, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_4 : __fmaf_rn(dl_1, _rcp_174, e2_4)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2333,12 +2368,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_5 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * e2_5); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_5, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_5, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_5 : __fmaf_rn(dl_3, _rcp_178, e2_5)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2362,12 +2403,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_6 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * e2_6); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_6, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_6, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_6 : __fmaf_rn(dl_5, _rcp_182, e2_6)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2403,12 +2450,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_2 = 48 + 8 * th + 2; @@ -2428,12 +2481,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_1 = approx_exp2(dc2_2.x); float e2_1_1 = approx_exp2(dc2_2.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_190 = approx_rcp(k_t0_2); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_191 = approx_rcp(k_t1_2); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_2.x; pan_641[3] = p2_2.y; int t0_3 = 48 + 8 * th + 4; @@ -2453,12 +2512,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_2 = approx_exp2(dc2_3.x); float e2_1_2 = approx_exp2(dc2_3.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_194 = approx_rcp(k_t0_3); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_195 = approx_rcp(k_t1_3); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_3.x; pan_641[5] = p2_3.y; int t0_4 = 48 + 8 * th + 6; @@ -2478,12 +2543,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(_f2_61, make_float2(g_t0_4, g_t1_4)); float e2_0_3 = approx_exp2(dc2_4.x); float e2_1_3 = approx_exp2(dc2_4.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_198 = approx_rcp(k_t0_4); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_199 = approx_rcp(k_t1_4); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_4.x; pan_641[7] = p2_4.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2531,12 +2602,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_202 = approx_rcp(k_t0_5); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_203 = approx_rcp(k_t1_5); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_5.x; pan_703[1] = p2_5.y; int t0_6 = 8 * th + 2; @@ -2560,12 +2637,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_2.x); float en_1_1 = approx_exp2(dn2_2.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_206 = approx_rcp(k_t0_6); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_207 = approx_rcp(k_t1_6); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_6.x; pan_703[3] = p2_6.y; int t0_7 = 8 * th + 4; @@ -2589,12 +2672,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_3.x); float en_1_2 = approx_exp2(dn2_3.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_210 = approx_rcp(k_t0_7); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_211 = approx_rcp(k_t1_7); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_7.x; pan_703[5] = p2_7.y; int t0_8 = 8 * th + 6; @@ -2618,12 +2707,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_4.x); float en_1_3 = approx_exp2(dn2_4.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_214 = approx_rcp(k_t0_8); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_215 = approx_rcp(k_t1_8); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_8.x; pan_703[7] = p2_8.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2665,12 +2760,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_5.x); float en_1_4 = approx_exp2(dn2_5.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_218 = approx_rcp(k_t0_9); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_219 = approx_rcp(k_t1_9); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_9.x; pan_777[1] = p2_9.y; int t0_10 = 16 + 8 * th + 2; @@ -2694,12 +2795,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_6.x); float en_1_5 = approx_exp2(dn2_6.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_222 = approx_rcp(k_t0_10); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_223 = approx_rcp(k_t1_10); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_10.x; pan_777[3] = p2_10.y; int t0_11 = 16 + 8 * th + 4; @@ -2723,12 +2830,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_7.x); float en_1_6 = approx_exp2(dn2_7.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_226 = approx_rcp(k_t0_11); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_227 = approx_rcp(k_t1_11); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_11.x; pan_777[5] = p2_11.y; int t0_12 = 16 + 8 * th + 6; @@ -2752,12 +2865,18 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 dn2_8 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_8.x); float en_1_7 = approx_exp2(dn2_8.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_230 = approx_rcp(k_t0_12); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_231 = approx_rcp(k_t1_12); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_12.x; pan_777[7] = p2_12.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2828,37 +2947,37 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_4 = approx_exp2(d2_57.x); float e1_4 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_4, e1_4); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_4, e1_4); float2 v2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_4.x; v_562_0[1] = v2_4.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_5 = approx_exp2(d2_58.x); float e1_5 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_5, e1_5); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_5, e1_5); float2 v2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_5.x; v_562_0[3] = v2_5.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_6 = approx_exp2(d2_59.x); float e1_6 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_6, e1_6); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_6, e1_6); float2 v2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_6.x; v_562_0[5] = v2_6.y; float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_7 = approx_exp2(d2_60.x); float e1_7 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_7, e1_7); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_7, e1_7); float2 v2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_7.x; v_562_0[7] = v2_7.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2870,29 +2989,29 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2900,29 +3019,29 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2992,7 +3111,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3024,7 +3148,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3056,7 +3185,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_2)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3088,7 +3222,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_3)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3120,7 +3259,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_4)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3152,7 +3296,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_5)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3184,7 +3333,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_6)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3216,7 +3370,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_7)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3231,31 +3390,31 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_216 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_216; + float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_248; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_217 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_217; + float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_249; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_218 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_218; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_250; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_219 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_219; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_251; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_220 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_220; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_252; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_221 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_221; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_253; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_222 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_222; - float _shfl_xor_sync_223 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_223; - float _shfl_xor_sync_224 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_224; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_254; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_255; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_256; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); int ew_0 = 98304 - 32768 * th >> 2; @@ -3267,8 +3426,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int qq = lane_1 >> 1 & 3; int j = qd + 4 * tb; bool live = 32 + 8 * th + j < clen; - int _min_225 = ((32 + 8 * th + j) < (clen - 1) ? (32 + 8 * th + j) : (clen - 1)); - long long row = ext0 + (long long)_min_225; + int _min_257 = ((32 + 8 * th + j) < (clen - 1) ? (32 + 8 * th + j) : (clen - 1)); + long long row = ext0 + (long long)_min_257; bool is_k = tk == 1; __nv_bfloat16* xr = ((is_k) ? (k_raw) : (q_raw)); int xts = ((is_k) ? k_raw_row_stride : q_raw_row_stride); @@ -3295,43 +3454,43 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (sb + 8 * (3 ^ qq))))); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint2 _vld_226; - _vld_226 = *reinterpret_cast(xr + (xrow + (long long)(8 * (0 ^ qq))) + 0); - uint32_t* _vpairs_226 = reinterpret_cast(&_vld_226); + uint2 _vld_258; + _vld_258 = *reinterpret_cast(xr + (xrow + (long long)(8 * (0 ^ qq))) + 0); + uint32_t* _vpairs_258 = reinterpret_cast(&_vld_258); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_226[_pair]) << 16); - (&xv[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_226[_pair]) & 0xffff0000u); + (&xv[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_258[_pair]) << 16); + (&xv[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_258[_pair]) & 0xffff0000u); } } { - uint2 _vld_227; - _vld_227 = *reinterpret_cast(xr + (xrow + (long long)(8 * (1 ^ qq))) + 0); - uint32_t* _vpairs_227 = reinterpret_cast(&_vld_227); + uint2 _vld_259; + _vld_259 = *reinterpret_cast(xr + (xrow + (long long)(8 * (1 ^ qq))) + 0); + uint32_t* _vpairs_259 = reinterpret_cast(&_vld_259); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_227[_pair]) << 16); - (&xv[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_227[_pair]) & 0xffff0000u); + (&xv[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_259[_pair]) << 16); + (&xv[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_259[_pair]) & 0xffff0000u); } } { - uint2 _vld_228; - _vld_228 = *reinterpret_cast(xr + (xrow + (long long)(8 * (2 ^ qq))) + 0); - uint32_t* _vpairs_228 = reinterpret_cast(&_vld_228); + uint2 _vld_260; + _vld_260 = *reinterpret_cast(xr + (xrow + (long long)(8 * (2 ^ qq))) + 0); + uint32_t* _vpairs_260 = reinterpret_cast(&_vld_260); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_228[_pair]) << 16); - (&xv[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_228[_pair]) & 0xffff0000u); + (&xv[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_260[_pair]) << 16); + (&xv[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_260[_pair]) & 0xffff0000u); } } { - uint2 _vld_229; - _vld_229 = *reinterpret_cast(xr + (xrow + (long long)(8 * (3 ^ qq))) + 0); - uint32_t* _vpairs_229 = reinterpret_cast(&_vld_229); + uint2 _vld_261; + _vld_261 = *reinterpret_cast(xr + (xrow + (long long)(8 * (3 ^ qq))) + 0); + uint32_t* _vpairs_261 = reinterpret_cast(&_vld_261); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_229[_pair]) << 16); - (&xv[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_229[_pair]) & 0xffff0000u); + (&xv[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_261[_pair]) << 16); + (&xv[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_261[_pair]) & 0xffff0000u); } } { @@ -3340,65 +3499,65 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 r2 = make_float2(rv[0], rv[0]); float y[16]; float p[16]; - float2 _f32x2_mul_a_230 = make_float2(xv[0], xv[1]); + float2 _f32x2_mul_a_262 = make_float2(xv[0], xv[1]); float2 y2_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_230), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_262), "l"(*(const unsigned long long*)&r2)); y[0] = y2_0.x; y[1] = y2_0.y; float2 p2_13 = fma_f32x2_rn_noftz(make_float2(g[0], g[1]), y2_0, nz2); p[0] = p2_13.x; p[1] = p2_13.y; - float2 _f32x2_mul_a_231 = make_float2(xv[2], xv[3]); + float2 _f32x2_mul_a_263 = make_float2(xv[2], xv[3]); float2 y2_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_231), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_263), "l"(*(const unsigned long long*)&r2)); y[2] = y2_1.x; y[3] = y2_1.y; float2 p2_14 = fma_f32x2_rn_noftz(make_float2(g[2], g[3]), y2_1, nz2); p[2] = p2_14.x; p[3] = p2_14.y; - float2 _f32x2_mul_a_232 = make_float2(xv[4], xv[5]); + float2 _f32x2_mul_a_264 = make_float2(xv[4], xv[5]); float2 y2_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_264), "l"(*(const unsigned long long*)&r2)); y[4] = y2_2.x; y[5] = y2_2.y; float2 p2_15 = fma_f32x2_rn_noftz(make_float2(g[4], g[5]), y2_2, nz2); p[4] = p2_15.x; p[5] = p2_15.y; - float2 _f32x2_mul_a_233 = make_float2(xv[6], xv[7]); + float2 _f32x2_mul_a_265 = make_float2(xv[6], xv[7]); float2 y2_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_233), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_265), "l"(*(const unsigned long long*)&r2)); y[6] = y2_3.x; y[7] = y2_3.y; float2 p2_16 = fma_f32x2_rn_noftz(make_float2(g[6], g[7]), y2_3, nz2); p[6] = p2_16.x; p[7] = p2_16.y; - float2 _f32x2_mul_a_234 = make_float2(xv[8], xv[9]); + float2 _f32x2_mul_a_266 = make_float2(xv[8], xv[9]); float2 y2_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_266), "l"(*(const unsigned long long*)&r2)); y[8] = y2_4.x; y[9] = y2_4.y; float2 p2_17 = fma_f32x2_rn_noftz(make_float2(g[8], g[9]), y2_4, nz2); p[8] = p2_17.x; p[9] = p2_17.y; - float2 _f32x2_mul_a_235 = make_float2(xv[10], xv[11]); + float2 _f32x2_mul_a_267 = make_float2(xv[10], xv[11]); float2 y2_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_235), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&r2)); y[10] = y2_5.x; y[11] = y2_5.y; float2 p2_18 = fma_f32x2_rn_noftz(make_float2(g[10], g[11]), y2_5, nz2); p[10] = p2_18.x; p[11] = p2_18.y; - float2 _f32x2_mul_a_236 = make_float2(xv[12], xv[13]); + float2 _f32x2_mul_a_268 = make_float2(xv[12], xv[13]); float2 y2_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_268), "l"(*(const unsigned long long*)&r2)); y[12] = y2_6.x; y[13] = y2_6.y; float2 p2_19 = fma_f32x2_rn_noftz(make_float2(g[12], g[13]), y2_6, nz2); p[12] = p2_19.x; p[13] = p2_19.y; - float2 _f32x2_mul_a_237 = make_float2(xv[14], xv[15]); + float2 _f32x2_mul_a_269 = make_float2(xv[14], xv[15]); float2 y2_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_237), "l"(*(const unsigned long long*)&r2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_269), "l"(*(const unsigned long long*)&r2)); y[14] = y2_7.x; y[15] = y2_7.y; float2 p2_20 = fma_f32x2_rn_noftz(make_float2(g[14], g[15]), y2_7, nz2); @@ -3415,35 +3574,35 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 e2_8 = add_f32x2_noftz(a2_1, c2_1); e[2] = e2_8.x; e[3] = e2_8.y; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, e[0], 1); - float _add_238; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_238) : "f"(e[0]), "f"(_shfl_xor_sync_239)); - e[0] = _add_238; - float _shfl_xor_sync_241 = __shfl_xor_sync(0xFFFFFFFF, e[1], 1); - float _add_240; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_240) : "f"(e[1]), "f"(_shfl_xor_sync_241)); - e[1] = _add_240; - float _shfl_xor_sync_243 = __shfl_xor_sync(0xFFFFFFFF, e[2], 1); - float _add_242; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_242) : "f"(e[2]), "f"(_shfl_xor_sync_243)); - e[2] = _add_242; - float _shfl_xor_sync_245 = __shfl_xor_sync(0xFFFFFFFF, e[3], 1); - float _add_244; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_244) : "f"(e[3]), "f"(_shfl_xor_sync_245)); - e[3] = _add_244; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, e[0], 1); + float _add_270; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_270) : "f"(e[0]), "f"(_shfl_xor_sync_271)); + e[0] = _add_270; + float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, e[1], 1); + float _add_272; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_272) : "f"(e[1]), "f"(_shfl_xor_sync_273)); + e[1] = _add_272; + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, e[2], 1); + float _add_274; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_274) : "f"(e[2]), "f"(_shfl_xor_sync_275)); + e[2] = _add_274; + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, e[3], 1); + float _add_276; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_276) : "f"(e[3]), "f"(_shfl_xor_sync_277)); + e[3] = _add_276; float2 f2 = add_f32x2_noftz(make_float2(e[0], e[1]), make_float2(e[2], e[3])); float sq; asm("add.rn.f32 %0, %1, %2;" : "=f"(sq) : "f"(f2.x), "f"(f2.y)); - float _shfl_sync_248 = __shfl_sync(0xFFFFFFFF, sq, 0, 8); - float _shfl_sync_249 = __shfl_sync(0xFFFFFFFF, sq, 2, 8); - float _add_247; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_247) : "f"(_shfl_sync_248), "f"(_shfl_sync_249)); - float _shfl_sync_250 = __shfl_sync(0xFFFFFFFF, sq, 4, 8); - float _add_246; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_246) : "f"(_add_247), "f"(_shfl_sync_250)); - float _shfl_sync_251 = __shfl_sync(0xFFFFFFFF, sq, 6, 8); + float _shfl_sync_280 = __shfl_sync(0xFFFFFFFF, sq, 0, 8); + float _shfl_sync_281 = __shfl_sync(0xFFFFFFFF, sq, 2, 8); + float _add_279; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_279) : "f"(_shfl_sync_280), "f"(_shfl_sync_281)); + float _shfl_sync_282 = __shfl_sync(0xFFFFFFFF, sq, 4, 8); + float _add_278; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_278) : "f"(_add_279), "f"(_shfl_sync_282)); + float _shfl_sync_283 = __shfl_sync(0xFFFFFFFF, sq, 6, 8); float dot; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_246), "f"(_shfl_sync_251)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_278), "f"(_shfl_sync_283)); float2 d2 = make_float2(dot, dot); float o_0[4]; float2 t2_0 = fma_f32x2_rn_noftz(make_float2(y[0], y[1]), d2, nz2); @@ -3581,7 +3740,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; + float qe_7 = q_t_1_8 * e_dg_0; + float kp_19 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_0; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_14[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_0) * _tmem_load_15[7]); + float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_16[7], k_t_0, cr_8)) + dgf_0; s_run_1059[0] = s_run_1059[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3613,7 +3777,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[6]; - float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, (dk2b_2 - _tmem_load_16[6]) * k_t_2) + dgf_2; + float qe_8 = q_t_1_9 * e_dg_2; + float kp_20 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_14[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_2) * _tmem_load_15[6]); + float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_16[6], k_t_2, cr_9)) + dgf_2; s_run_1059[0] = s_run_1059[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3645,7 +3814,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[5]; - float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, (dk2b_3 - _tmem_load_16[5]) * k_t_3) + dgf_3; + float qe_9 = q_t_1_10 * e_dg_3; + float kp_21 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_3; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_14[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_3) * _tmem_load_15[5]); + float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_16[5], k_t_3, cr_10)) + dgf_3; s_run_1059[0] = s_run_1059[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3677,7 +3851,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[4]; - float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, (dk2b_4 - _tmem_load_16[4]) * k_t_4) + dgf_4; + float qe_10 = q_t_1_11 * e_dg_4; + float kp_22 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_4; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_14[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_4) * _tmem_load_15[4]); + float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_16[4], k_t_4, cr_11)) + dgf_4; s_run_1059[0] = s_run_1059[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3709,7 +3888,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[3]; - float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, (dk2b_5 - _tmem_load_16[3]) * k_t_5) + dgf_5; + float qe_11 = q_t_1_12 * e_dg_5; + float kp_23 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_5; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_14[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_5) * _tmem_load_15[3]); + float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_16[3], k_t_5, cr_12)) + dgf_5; s_run_1059[0] = s_run_1059[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3741,7 +3925,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[2]; - float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, (dk2b_6 - _tmem_load_16[2]) * k_t_6) + dgf_6; + float qe_12 = q_t_1_13 * e_dg_6; + float kp_24 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_6; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_14[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_6) * _tmem_load_15[2]); + float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_16[2], k_t_6, cr_13)) + dgf_6; s_run_1059[0] = s_run_1059[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3773,7 +3962,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[1]; - float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, (dk2b_7 - _tmem_load_16[1]) * k_t_7) + dgf_7; + float qe_13 = q_t_1_14 * e_dg_7; + float kp_25 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_7; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_14[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_7) * _tmem_load_15[1]); + float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_16[1], k_t_7, cr_14)) + dgf_7; s_run_1059[0] = s_run_1059[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3805,7 +3999,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_16[0]; - float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, (dk2b_8 - _tmem_load_16[0]) * k_t_8) + dgf_8; + float qe_14 = q_t_1_15 * e_dg_8; + float kp_26 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_8; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_14[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_8) * _tmem_load_15[0]); + float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_16[0], k_t_8, cr_15)) + dgf_8; s_run_1059[0] = s_run_1059[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3820,31 +4019,31 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int b2_1231 = lane_1 >> 2 & 1; float v4_1232[4]; float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_252; + float _shfl_xor_sync_284 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_284; float send_2 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_253; + float _shfl_xor_sync_285 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_285; float send_3 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_254; + float _shfl_xor_sync_286 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_286; float send_4 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_255; + float _shfl_xor_sync_287 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_287; float v2_1237[2]; float send_5 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_256; + float _shfl_xor_sync_288 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_288; float send_6 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); + v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_289; float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_258; - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_259; - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_260; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); + float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_290; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); + v1_1241 = v1_1241 + _shfl_xor_sync_291; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); + v1_1241 = v1_1241 + _shfl_xor_sync_292; asm volatile("tcgen05.fence::before_thread_sync;"); int ew_1 = 98304 - 32768 * th >> 2; float nz_0 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); @@ -3855,8 +4054,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int qq_0 = lane_1 >> 1 & 3; int j_0 = qd + 4 * tb_0; bool live_0 = 48 + 8 * th + j_0 < clen; - int _min_261 = ((48 + 8 * th + j_0) < (clen - 1) ? (48 + 8 * th + j_0) : (clen - 1)); - long long row_0 = ext0 + (long long)_min_261; + int _min_293 = ((48 + 8 * th + j_0) < (clen - 1) ? (48 + 8 * th + j_0) : (clen - 1)); + long long row_0 = ext0 + (long long)_min_293; bool is_k_0 = tk_0 == 1; __nv_bfloat16* xr_0 = ((is_k_0) ? (k_raw) : (q_raw)); int xts_0 = ((is_k_0) ? k_raw_row_stride : q_raw_row_stride); @@ -3883,43 +4082,43 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (sb_0 + 8 * (3 ^ qq_0))))); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint2 _vld_262; - _vld_262 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (0 ^ qq_0))) + 0); - uint32_t* _vpairs_262 = reinterpret_cast(&_vld_262); + uint2 _vld_294; + _vld_294 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (0 ^ qq_0))) + 0); + uint32_t* _vpairs_294 = reinterpret_cast(&_vld_294); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_262[_pair]) << 16); - (&xv_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_262[_pair]) & 0xffff0000u); + (&xv_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_294[_pair]) << 16); + (&xv_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_294[_pair]) & 0xffff0000u); } } { - uint2 _vld_263; - _vld_263 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (1 ^ qq_0))) + 0); - uint32_t* _vpairs_263 = reinterpret_cast(&_vld_263); + uint2 _vld_295; + _vld_295 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (1 ^ qq_0))) + 0); + uint32_t* _vpairs_295 = reinterpret_cast(&_vld_295); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_0[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_263[_pair]) << 16); - (&xv_0[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_263[_pair]) & 0xffff0000u); + (&xv_0[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_295[_pair]) << 16); + (&xv_0[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_295[_pair]) & 0xffff0000u); } } { - uint2 _vld_264; - _vld_264 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (2 ^ qq_0))) + 0); - uint32_t* _vpairs_264 = reinterpret_cast(&_vld_264); + uint2 _vld_296; + _vld_296 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (2 ^ qq_0))) + 0); + uint32_t* _vpairs_296 = reinterpret_cast(&_vld_296); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_0[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_264[_pair]) << 16); - (&xv_0[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_264[_pair]) & 0xffff0000u); + (&xv_0[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_296[_pair]) << 16); + (&xv_0[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_296[_pair]) & 0xffff0000u); } } { - uint2 _vld_265; - _vld_265 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (3 ^ qq_0))) + 0); - uint32_t* _vpairs_265 = reinterpret_cast(&_vld_265); + uint2 _vld_297; + _vld_297 = *reinterpret_cast(xr_0 + (xrow_0 + (long long)(8 * (3 ^ qq_0))) + 0); + uint32_t* _vpairs_297 = reinterpret_cast(&_vld_297); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_0[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_265[_pair]) << 16); - (&xv_0[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_265[_pair]) & 0xffff0000u); + (&xv_0[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_297[_pair]) << 16); + (&xv_0[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_297[_pair]) & 0xffff0000u); } } { @@ -3928,65 +4127,65 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 r2_0 = make_float2(rv_0[0], rv_0[0]); float y_0[16]; float p_0[16]; - float2 _f32x2_mul_a_266 = make_float2(xv_0[0], xv_0[1]); + float2 _f32x2_mul_a_298 = make_float2(xv_0[0], xv_0[1]); float2 y2_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_266), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_298), "l"(*(const unsigned long long*)&r2_0)); y_0[0] = y2_8.x; y_0[1] = y2_8.y; float2 p2_21 = fma_f32x2_rn_noftz(make_float2(g_0[0], g_0[1]), y2_8, nz2_0); p_0[0] = p2_21.x; p_0[1] = p2_21.y; - float2 _f32x2_mul_a_267 = make_float2(xv_0[2], xv_0[3]); + float2 _f32x2_mul_a_299 = make_float2(xv_0[2], xv_0[3]); float2 y2_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_299), "l"(*(const unsigned long long*)&r2_0)); y_0[2] = y2_9.x; y_0[3] = y2_9.y; float2 p2_22 = fma_f32x2_rn_noftz(make_float2(g_0[2], g_0[3]), y2_9, nz2_0); p_0[2] = p2_22.x; p_0[3] = p2_22.y; - float2 _f32x2_mul_a_268 = make_float2(xv_0[4], xv_0[5]); + float2 _f32x2_mul_a_300 = make_float2(xv_0[4], xv_0[5]); float2 y2_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_268), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&r2_0)); y_0[4] = y2_10.x; y_0[5] = y2_10.y; float2 p2_23 = fma_f32x2_rn_noftz(make_float2(g_0[4], g_0[5]), y2_10, nz2_0); p_0[4] = p2_23.x; p_0[5] = p2_23.y; - float2 _f32x2_mul_a_269 = make_float2(xv_0[6], xv_0[7]); + float2 _f32x2_mul_a_301 = make_float2(xv_0[6], xv_0[7]); float2 y2_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_269), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_301), "l"(*(const unsigned long long*)&r2_0)); y_0[6] = y2_11.x; y_0[7] = y2_11.y; float2 p2_24 = fma_f32x2_rn_noftz(make_float2(g_0[6], g_0[7]), y2_11, nz2_0); p_0[6] = p2_24.x; p_0[7] = p2_24.y; - float2 _f32x2_mul_a_270 = make_float2(xv_0[8], xv_0[9]); + float2 _f32x2_mul_a_302 = make_float2(xv_0[8], xv_0[9]); float2 y2_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_270), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_302), "l"(*(const unsigned long long*)&r2_0)); y_0[8] = y2_12.x; y_0[9] = y2_12.y; float2 p2_25 = fma_f32x2_rn_noftz(make_float2(g_0[8], g_0[9]), y2_12, nz2_0); p_0[8] = p2_25.x; p_0[9] = p2_25.y; - float2 _f32x2_mul_a_271 = make_float2(xv_0[10], xv_0[11]); + float2 _f32x2_mul_a_303 = make_float2(xv_0[10], xv_0[11]); float2 y2_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_271), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_303), "l"(*(const unsigned long long*)&r2_0)); y_0[10] = y2_13.x; y_0[11] = y2_13.y; float2 p2_26 = fma_f32x2_rn_noftz(make_float2(g_0[10], g_0[11]), y2_13, nz2_0); p_0[10] = p2_26.x; p_0[11] = p2_26.y; - float2 _f32x2_mul_a_272 = make_float2(xv_0[12], xv_0[13]); + float2 _f32x2_mul_a_304 = make_float2(xv_0[12], xv_0[13]); float2 y2_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_272), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_304), "l"(*(const unsigned long long*)&r2_0)); y_0[12] = y2_14.x; y_0[13] = y2_14.y; float2 p2_27 = fma_f32x2_rn_noftz(make_float2(g_0[12], g_0[13]), y2_14, nz2_0); p_0[12] = p2_27.x; p_0[13] = p2_27.y; - float2 _f32x2_mul_a_273 = make_float2(xv_0[14], xv_0[15]); + float2 _f32x2_mul_a_305 = make_float2(xv_0[14], xv_0[15]); float2 y2_15; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_273), "l"(*(const unsigned long long*)&r2_0)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_305), "l"(*(const unsigned long long*)&r2_0)); y_0[14] = y2_15.x; y_0[15] = y2_15.y; float2 p2_28 = fma_f32x2_rn_noftz(make_float2(g_0[14], g_0[15]), y2_15, nz2_0); @@ -4003,35 +4202,35 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 e2_10 = add_f32x2_noftz(a2_3, c2_3); e_0[2] = e2_10.x; e_0[3] = e2_10.y; - float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, e_0[0], 1); - float _add_274; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_274) : "f"(e_0[0]), "f"(_shfl_xor_sync_275)); - e_0[0] = _add_274; - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, e_0[1], 1); - float _add_276; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_276) : "f"(e_0[1]), "f"(_shfl_xor_sync_277)); - e_0[1] = _add_276; - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, e_0[2], 1); - float _add_278; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_278) : "f"(e_0[2]), "f"(_shfl_xor_sync_279)); - e_0[2] = _add_278; - float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, e_0[3], 1); - float _add_280; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_280) : "f"(e_0[3]), "f"(_shfl_xor_sync_281)); - e_0[3] = _add_280; + float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, e_0[0], 1); + float _add_306; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_306) : "f"(e_0[0]), "f"(_shfl_xor_sync_307)); + e_0[0] = _add_306; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, e_0[1], 1); + float _add_308; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_308) : "f"(e_0[1]), "f"(_shfl_xor_sync_309)); + e_0[1] = _add_308; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, e_0[2], 1); + float _add_310; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_310) : "f"(e_0[2]), "f"(_shfl_xor_sync_311)); + e_0[2] = _add_310; + float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, e_0[3], 1); + float _add_312; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_312) : "f"(e_0[3]), "f"(_shfl_xor_sync_313)); + e_0[3] = _add_312; float2 f2_0 = add_f32x2_noftz(make_float2(e_0[0], e_0[1]), make_float2(e_0[2], e_0[3])); float sq_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_0) : "f"(f2_0.x), "f"(f2_0.y)); - float _shfl_sync_284 = __shfl_sync(0xFFFFFFFF, sq_0, 0, 8); - float _shfl_sync_285 = __shfl_sync(0xFFFFFFFF, sq_0, 2, 8); - float _add_283; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_283) : "f"(_shfl_sync_284), "f"(_shfl_sync_285)); - float _shfl_sync_286 = __shfl_sync(0xFFFFFFFF, sq_0, 4, 8); - float _add_282; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_282) : "f"(_add_283), "f"(_shfl_sync_286)); - float _shfl_sync_287 = __shfl_sync(0xFFFFFFFF, sq_0, 6, 8); + float _shfl_sync_316 = __shfl_sync(0xFFFFFFFF, sq_0, 0, 8); + float _shfl_sync_317 = __shfl_sync(0xFFFFFFFF, sq_0, 2, 8); + float _add_315; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_315) : "f"(_shfl_sync_316), "f"(_shfl_sync_317)); + float _shfl_sync_318 = __shfl_sync(0xFFFFFFFF, sq_0, 4, 8); + float _add_314; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_314) : "f"(_add_315), "f"(_shfl_sync_318)); + float _shfl_sync_319 = __shfl_sync(0xFFFFFFFF, sq_0, 6, 8); float dot_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_282), "f"(_shfl_sync_287)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_314), "f"(_shfl_sync_319)); float2 d2_61 = make_float2(dot_0, dot_0); float o_4[4]; float2 t2_8 = fma_f32x2_rn_noftz(make_float2(y_0[0], y_0[1]), d2_61, nz2_0); @@ -4114,8 +4313,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM } asm volatile("{ .reg .pred p_; setp.ne.b32 p_, %3, 0; @p_ st.global.v2.b32 [%0], {%1, %2}; }" :: "l"(dxp_0 + (orow_0 + (long long)(8 * (3 ^ qq_0)))), "r"(ob_7[(0) + 0]), "r"(ob_7[(0) + 1]), "r"((unsigned int)(live_0)) : "memory"); { - uint32_t _addr_288 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_288), "f"(tot_1228) : "memory"); + uint32_t _addr_320 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_320), "f"(tot_1228) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4144,9 +4343,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_0 = offset + _tmem_load_17[0]; if (t_9 < clen) { { - auto _gst_289 = dg_base + goff_1_0; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast(_gst_289) + (0)) = dg_1_0; + auto _gst_321 = dg_base + goff_1_0; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast(_gst_321) + (0)) = dg_1_0; } } int t_10 = 48 + 8 * th + 1; @@ -4154,9 +4353,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_1 = offset + _tmem_load_17[1]; if (t_10 < clen) { { - auto _gst_290 = dg_base + goff_1_1; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast(_gst_290) + (0)) = dg_1_1; + auto _gst_322 = dg_base + goff_1_1; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast(_gst_322) + (0)) = dg_1_1; } } int t_11 = 48 + 8 * th + 2; @@ -4164,9 +4363,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_2 = offset + _tmem_load_17[2]; if (t_11 < clen) { { - auto _gst_291 = dg_base + goff_1_2; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast(_gst_291) + (0)) = dg_1_2; + auto _gst_323 = dg_base + goff_1_2; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast(_gst_323) + (0)) = dg_1_2; } } int t_12 = 48 + 8 * th + 3; @@ -4174,9 +4373,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_3 = offset + _tmem_load_17[3]; if (t_12 < clen) { { - auto _gst_292 = dg_base + goff_1_3; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast(_gst_292) + (0)) = dg_1_3; + auto _gst_324 = dg_base + goff_1_3; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast(_gst_324) + (0)) = dg_1_3; } } int t_13 = 48 + 8 * th + 4; @@ -4184,9 +4383,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_4 = offset + _tmem_load_17[4]; if (t_13 < clen) { { - auto _gst_293 = dg_base + goff_1_4; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast(_gst_293) + (0)) = dg_1_4; + auto _gst_325 = dg_base + goff_1_4; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast(_gst_325) + (0)) = dg_1_4; } } int t_14 = 48 + 8 * th + 5; @@ -4194,9 +4393,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_5 = offset + _tmem_load_17[5]; if (t_14 < clen) { { - auto _gst_294 = dg_base + goff_1_5; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast(_gst_294) + (0)) = dg_1_5; + auto _gst_326 = dg_base + goff_1_5; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast(_gst_326) + (0)) = dg_1_5; } } int t_15 = 48 + 8 * th + 6; @@ -4204,9 +4403,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_6 = offset + _tmem_load_17[6]; if (t_15 < clen) { { - auto _gst_295 = dg_base + goff_1_6; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast(_gst_295) + (0)) = dg_1_6; + auto _gst_327 = dg_base + goff_1_6; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast(_gst_327) + (0)) = dg_1_6; } } int t_16 = 48 + 8 * th + 7; @@ -4214,9 +4413,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1_7 = offset + _tmem_load_17[7]; if (t_16 < clen) { { - auto _gst_296 = dg_base + goff_1_7; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast(_gst_296) + (0)) = dg_1_7; + auto _gst_328 = dg_base + goff_1_7; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast(_gst_328) + (0)) = dg_1_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -4234,9 +4433,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_0 = offset_1250 + _tmem_load_19[0]; if (t_17 < clen) { { - auto _gst_297 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast(_gst_297) + (0)) = dg_0; + auto _gst_329 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast(_gst_329) + (0)) = dg_0; } } int t_18 = 32 + 8 * th + 1; @@ -4244,9 +4443,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_1 = offset_1250 + _tmem_load_19[1]; if (t_18 < clen) { { - auto _gst_298 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast(_gst_298) + (0)) = dg_1; + auto _gst_330 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast(_gst_330) + (0)) = dg_1; } } int t_19 = 32 + 8 * th + 2; @@ -4254,9 +4453,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_2 = offset_1250 + _tmem_load_19[2]; if (t_19 < clen) { { - auto _gst_299 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast(_gst_299) + (0)) = dg_2; + auto _gst_331 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast(_gst_331) + (0)) = dg_2; } } int t_20 = 32 + 8 * th + 3; @@ -4264,9 +4463,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_3 = offset_1250 + _tmem_load_19[3]; if (t_20 < clen) { { - auto _gst_300 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_300)); - *(reinterpret_cast(_gst_300) + (0)) = dg_3; + auto _gst_332 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_332)); + *(reinterpret_cast(_gst_332) + (0)) = dg_3; } } int t_21 = 32 + 8 * th + 4; @@ -4274,9 +4473,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_4 = offset_1250 + _tmem_load_19[4]; if (t_21 < clen) { { - auto _gst_301 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_301)); - *(reinterpret_cast(_gst_301) + (0)) = dg_4; + auto _gst_333 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_333)); + *(reinterpret_cast(_gst_333) + (0)) = dg_4; } } int t_22 = 32 + 8 * th + 5; @@ -4284,9 +4483,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_5 = offset_1250 + _tmem_load_19[5]; if (t_22 < clen) { { - auto _gst_302 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_302)); - *(reinterpret_cast(_gst_302) + (0)) = dg_5; + auto _gst_334 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_334)); + *(reinterpret_cast(_gst_334) + (0)) = dg_5; } } int t_23 = 32 + 8 * th + 6; @@ -4294,9 +4493,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_6 = offset_1250 + _tmem_load_19[6]; if (t_23 < clen) { { - auto _gst_303 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_303)); - *(reinterpret_cast(_gst_303) + (0)) = dg_6; + auto _gst_335 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_335)); + *(reinterpret_cast(_gst_335) + (0)) = dg_6; } } int t_24 = 32 + 8 * th + 7; @@ -4304,15 +4503,15 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_7 = offset_1250 + _tmem_load_19[7]; if (t_24 < clen) { { - auto _gst_304 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_304)); - *(reinterpret_cast(_gst_304) + (0)) = dg_7; + auto _gst_336 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_336)); + *(reinterpret_cast(_gst_336) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi_1248 + t_lo_1249; { - uint32_t _addr_305 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_305), "f"(off_acc[0]) : "memory"); + uint32_t _addr_337 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_337), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4372,7 +4571,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[7]; - float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, (dk2b_9 - _tmem_load_24[7]) * k_t_9) + dgf_9; + float qe_15 = q_t_1_16 * e_dg_9; + float kp_27 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_9; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_9) * _tmem_load_23[7]); + float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_24[7], k_t_9, cr_16)) + dgf_9; s_run_1302[0] = s_run_1302[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4404,7 +4608,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[6]; - float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, (dk2b_10 - _tmem_load_24[6]) * k_t_10) + dgf_10; + float qe_16 = q_t_1_17 * e_dg_10; + float kp_28 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_10; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_10) * _tmem_load_23[6]); + float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_24[6], k_t_10, cr_17)) + dgf_10; s_run_1302[0] = s_run_1302[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4436,7 +4645,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[5]; - float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, (dk2b_11 - _tmem_load_24[5]) * k_t_11) + dgf_11; + float qe_17 = q_t_1_18 * e_dg_11; + float kp_29 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_11; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_11) * _tmem_load_23[5]); + float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_24[5], k_t_11, cr_18)) + dgf_11; s_run_1302[0] = s_run_1302[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4468,7 +4682,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[4]; - float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, (dk2b_12 - _tmem_load_24[4]) * k_t_12) + dgf_12; + float qe_18 = q_t_1_19 * e_dg_12; + float kp_30 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_12; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_12) * _tmem_load_23[4]); + float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_24[4], k_t_12, cr_19)) + dgf_12; s_run_1302[0] = s_run_1302[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4500,7 +4719,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[3]; - float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, (dk2b_13 - _tmem_load_24[3]) * k_t_13) + dgf_13; + float qe_19 = q_t_1_20 * e_dg_13; + float kp_31 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_13; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_13) * _tmem_load_23[3]); + float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_24[3], k_t_13, cr_20)) + dgf_13; s_run_1302[0] = s_run_1302[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4532,7 +4756,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[2]; - float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, (dk2b_14 - _tmem_load_24[2]) * k_t_14) + dgf_14; + float qe_20 = q_t_1_21 * e_dg_14; + float kp_32 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_14; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_14) * _tmem_load_23[2]); + float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_24[2], k_t_14, cr_21)) + dgf_14; s_run_1302[0] = s_run_1302[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4564,7 +4793,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[1]; - float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, (dk2b_15 - _tmem_load_24[1]) * k_t_15) + dgf_15; + float qe_21 = q_t_1_22 * e_dg_15; + float kp_33 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_15; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_15) * _tmem_load_23[1]); + float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_24[1], k_t_15, cr_22)) + dgf_15; s_run_1302[0] = s_run_1302[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4596,7 +4830,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_24[0]; - float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, (dk2b_16 - _tmem_load_24[0]) * k_t_16) + dgf_16; + float qe_22 = q_t_1_23 * e_dg_16; + float kp_34 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_16; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_16) * _tmem_load_23[0]); + float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, __fmaf_rn(dk2b_16 - _tmem_load_24[0], k_t_16, cr_23)) + dgf_16; s_run_1302[0] = s_run_1302[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4611,31 +4850,31 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; float send_7 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_306; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_338; float send_8 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_307; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_339; float send_9 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_308; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_340; float send_10 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_309; + float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_341; float v2_1480[2]; float send_11 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_310; + float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_342; float send_12 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_311; + float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_343; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_312; - float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_313; - float _shfl_xor_sync_314 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_314; + float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_344; + float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_345; + float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_346; asm volatile("tcgen05.fence::before_thread_sync;"); int ew_2 = 81920 - 32768 * th >> 2; float nz_1 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); @@ -4646,8 +4885,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int qq_1 = lane_1 >> 1 & 3; int j_1 = qd + 4 * tb_1; bool live_1 = 16 + 8 * th + j_1 < clen; - int _min_315 = ((16 + 8 * th + j_1) < (clen - 1) ? (16 + 8 * th + j_1) : (clen - 1)); - long long row_1 = ext0 + (long long)_min_315; + int _min_347 = ((16 + 8 * th + j_1) < (clen - 1) ? (16 + 8 * th + j_1) : (clen - 1)); + long long row_1 = ext0 + (long long)_min_347; bool is_k_1 = tk_1 == 1; __nv_bfloat16* xr_1 = ((is_k_1) ? (k_raw) : (q_raw)); int xts_1 = ((is_k_1) ? k_raw_row_stride : q_raw_row_stride); @@ -4674,43 +4913,43 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (sb_1 + 8 * (3 ^ qq_1))))); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); { - uint2 _vld_316; - _vld_316 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (0 ^ qq_1))) + 0); - uint32_t* _vpairs_316 = reinterpret_cast(&_vld_316); + uint2 _vld_348; + _vld_348 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (0 ^ qq_1))) + 0); + uint32_t* _vpairs_348 = reinterpret_cast(&_vld_348); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_316[_pair]) << 16); - (&xv_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_316[_pair]) & 0xffff0000u); + (&xv_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_348[_pair]) << 16); + (&xv_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_348[_pair]) & 0xffff0000u); } } { - uint2 _vld_317; - _vld_317 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (1 ^ qq_1))) + 0); - uint32_t* _vpairs_317 = reinterpret_cast(&_vld_317); + uint2 _vld_349; + _vld_349 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (1 ^ qq_1))) + 0); + uint32_t* _vpairs_349 = reinterpret_cast(&_vld_349); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_1[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_317[_pair]) << 16); - (&xv_1[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_317[_pair]) & 0xffff0000u); + (&xv_1[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_349[_pair]) << 16); + (&xv_1[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_349[_pair]) & 0xffff0000u); } } { - uint2 _vld_318; - _vld_318 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (2 ^ qq_1))) + 0); - uint32_t* _vpairs_318 = reinterpret_cast(&_vld_318); + uint2 _vld_350; + _vld_350 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (2 ^ qq_1))) + 0); + uint32_t* _vpairs_350 = reinterpret_cast(&_vld_350); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_1[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_318[_pair]) << 16); - (&xv_1[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_318[_pair]) & 0xffff0000u); + (&xv_1[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_350[_pair]) << 16); + (&xv_1[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_350[_pair]) & 0xffff0000u); } } { - uint2 _vld_319; - _vld_319 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (3 ^ qq_1))) + 0); - uint32_t* _vpairs_319 = reinterpret_cast(&_vld_319); + uint2 _vld_351; + _vld_351 = *reinterpret_cast(xr_1 + (xrow_1 + (long long)(8 * (3 ^ qq_1))) + 0); + uint32_t* _vpairs_351 = reinterpret_cast(&_vld_351); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_1[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_319[_pair]) << 16); - (&xv_1[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_319[_pair]) & 0xffff0000u); + (&xv_1[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_351[_pair]) << 16); + (&xv_1[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_351[_pair]) & 0xffff0000u); } } { @@ -4719,65 +4958,65 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 r2_1 = make_float2(rv_1[0], rv_1[0]); float y_1[16]; float p_1[16]; - float2 _f32x2_mul_a_320 = make_float2(xv_1[0], xv_1[1]); + float2 _f32x2_mul_a_352 = make_float2(xv_1[0], xv_1[1]); float2 y2_16; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_16) : "l"(*(const unsigned long long*)&_f32x2_mul_a_320), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_16) : "l"(*(const unsigned long long*)&_f32x2_mul_a_352), "l"(*(const unsigned long long*)&r2_1)); y_1[0] = y2_16.x; y_1[1] = y2_16.y; float2 p2_29 = fma_f32x2_rn_noftz(make_float2(g_1[0], g_1[1]), y2_16, nz2_1); p_1[0] = p2_29.x; p_1[1] = p2_29.y; - float2 _f32x2_mul_a_321 = make_float2(xv_1[2], xv_1[3]); + float2 _f32x2_mul_a_353 = make_float2(xv_1[2], xv_1[3]); float2 y2_17; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_17) : "l"(*(const unsigned long long*)&_f32x2_mul_a_321), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_17) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&r2_1)); y_1[2] = y2_17.x; y_1[3] = y2_17.y; float2 p2_30 = fma_f32x2_rn_noftz(make_float2(g_1[2], g_1[3]), y2_17, nz2_1); p_1[2] = p2_30.x; p_1[3] = p2_30.y; - float2 _f32x2_mul_a_322 = make_float2(xv_1[4], xv_1[5]); + float2 _f32x2_mul_a_354 = make_float2(xv_1[4], xv_1[5]); float2 y2_18; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_18) : "l"(*(const unsigned long long*)&_f32x2_mul_a_322), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_18) : "l"(*(const unsigned long long*)&_f32x2_mul_a_354), "l"(*(const unsigned long long*)&r2_1)); y_1[4] = y2_18.x; y_1[5] = y2_18.y; float2 p2_31 = fma_f32x2_rn_noftz(make_float2(g_1[4], g_1[5]), y2_18, nz2_1); p_1[4] = p2_31.x; p_1[5] = p2_31.y; - float2 _f32x2_mul_a_323 = make_float2(xv_1[6], xv_1[7]); + float2 _f32x2_mul_a_355 = make_float2(xv_1[6], xv_1[7]); float2 y2_19; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_19) : "l"(*(const unsigned long long*)&_f32x2_mul_a_323), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_19) : "l"(*(const unsigned long long*)&_f32x2_mul_a_355), "l"(*(const unsigned long long*)&r2_1)); y_1[6] = y2_19.x; y_1[7] = y2_19.y; float2 p2_32 = fma_f32x2_rn_noftz(make_float2(g_1[6], g_1[7]), y2_19, nz2_1); p_1[6] = p2_32.x; p_1[7] = p2_32.y; - float2 _f32x2_mul_a_324 = make_float2(xv_1[8], xv_1[9]); + float2 _f32x2_mul_a_356 = make_float2(xv_1[8], xv_1[9]); float2 y2_20; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_324), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_356), "l"(*(const unsigned long long*)&r2_1)); y_1[8] = y2_20.x; y_1[9] = y2_20.y; float2 p2_33 = fma_f32x2_rn_noftz(make_float2(g_1[8], g_1[9]), y2_20, nz2_1); p_1[8] = p2_33.x; p_1[9] = p2_33.y; - float2 _f32x2_mul_a_325 = make_float2(xv_1[10], xv_1[11]); + float2 _f32x2_mul_a_357 = make_float2(xv_1[10], xv_1[11]); float2 y2_21; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_325), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_357), "l"(*(const unsigned long long*)&r2_1)); y_1[10] = y2_21.x; y_1[11] = y2_21.y; float2 p2_34 = fma_f32x2_rn_noftz(make_float2(g_1[10], g_1[11]), y2_21, nz2_1); p_1[10] = p2_34.x; p_1[11] = p2_34.y; - float2 _f32x2_mul_a_326 = make_float2(xv_1[12], xv_1[13]); + float2 _f32x2_mul_a_358 = make_float2(xv_1[12], xv_1[13]); float2 y2_22; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_326), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_358), "l"(*(const unsigned long long*)&r2_1)); y_1[12] = y2_22.x; y_1[13] = y2_22.y; float2 p2_35 = fma_f32x2_rn_noftz(make_float2(g_1[12], g_1[13]), y2_22, nz2_1); p_1[12] = p2_35.x; p_1[13] = p2_35.y; - float2 _f32x2_mul_a_327 = make_float2(xv_1[14], xv_1[15]); + float2 _f32x2_mul_a_359 = make_float2(xv_1[14], xv_1[15]); float2 y2_23; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_327), "l"(*(const unsigned long long*)&r2_1)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_359), "l"(*(const unsigned long long*)&r2_1)); y_1[14] = y2_23.x; y_1[15] = y2_23.y; float2 p2_36 = fma_f32x2_rn_noftz(make_float2(g_1[14], g_1[15]), y2_23, nz2_1); @@ -4794,35 +5033,35 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 e2_12 = add_f32x2_noftz(a2_5, c2_5); e_1[2] = e2_12.x; e_1[3] = e2_12.y; - float _shfl_xor_sync_329 = __shfl_xor_sync(0xFFFFFFFF, e_1[0], 1); - float _add_328; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(e_1[0]), "f"(_shfl_xor_sync_329)); - e_1[0] = _add_328; - float _shfl_xor_sync_331 = __shfl_xor_sync(0xFFFFFFFF, e_1[1], 1); - float _add_330; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(e_1[1]), "f"(_shfl_xor_sync_331)); - e_1[1] = _add_330; - float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, e_1[2], 1); - float _add_332; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_332) : "f"(e_1[2]), "f"(_shfl_xor_sync_333)); - e_1[2] = _add_332; - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, e_1[3], 1); - float _add_334; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(e_1[3]), "f"(_shfl_xor_sync_335)); - e_1[3] = _add_334; + float _shfl_xor_sync_361 = __shfl_xor_sync(0xFFFFFFFF, e_1[0], 1); + float _add_360; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_360) : "f"(e_1[0]), "f"(_shfl_xor_sync_361)); + e_1[0] = _add_360; + float _shfl_xor_sync_363 = __shfl_xor_sync(0xFFFFFFFF, e_1[1], 1); + float _add_362; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_362) : "f"(e_1[1]), "f"(_shfl_xor_sync_363)); + e_1[1] = _add_362; + float _shfl_xor_sync_365 = __shfl_xor_sync(0xFFFFFFFF, e_1[2], 1); + float _add_364; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_364) : "f"(e_1[2]), "f"(_shfl_xor_sync_365)); + e_1[2] = _add_364; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, e_1[3], 1); + float _add_366; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_366) : "f"(e_1[3]), "f"(_shfl_xor_sync_367)); + e_1[3] = _add_366; float2 f2_1 = add_f32x2_noftz(make_float2(e_1[0], e_1[1]), make_float2(e_1[2], e_1[3])); float sq_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1) : "f"(f2_1.x), "f"(f2_1.y)); - float _shfl_sync_338 = __shfl_sync(0xFFFFFFFF, sq_1, 0, 8); - float _shfl_sync_339 = __shfl_sync(0xFFFFFFFF, sq_1, 2, 8); - float _add_337; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_337) : "f"(_shfl_sync_338), "f"(_shfl_sync_339)); - float _shfl_sync_340 = __shfl_sync(0xFFFFFFFF, sq_1, 4, 8); - float _add_336; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_336) : "f"(_add_337), "f"(_shfl_sync_340)); - float _shfl_sync_341 = __shfl_sync(0xFFFFFFFF, sq_1, 6, 8); + float _shfl_sync_370 = __shfl_sync(0xFFFFFFFF, sq_1, 0, 8); + float _shfl_sync_371 = __shfl_sync(0xFFFFFFFF, sq_1, 2, 8); + float _add_369; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_369) : "f"(_shfl_sync_370), "f"(_shfl_sync_371)); + float _shfl_sync_372 = __shfl_sync(0xFFFFFFFF, sq_1, 4, 8); + float _add_368; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_368) : "f"(_add_369), "f"(_shfl_sync_372)); + float _shfl_sync_373 = __shfl_sync(0xFFFFFFFF, sq_1, 6, 8); float dot_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_336), "f"(_shfl_sync_341)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_368), "f"(_shfl_sync_373)); float2 d2_62 = make_float2(dot_1, dot_1); float o_8[4]; float2 t2_16 = fma_f32x2_rn_noftz(make_float2(y_1[0], y_1[1]), d2_62, nz2_1); @@ -4906,8 +5145,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM asm volatile("{ .reg .pred p_; setp.ne.b32 p_, %3, 0; @p_ st.global.v2.b32 [%0], {%1, %2}; }" :: "l"(dxp_1 + (orow_1 + (long long)(8 * (3 ^ qq_1)))), "r"(ob_11[(0) + 0]), "r"(ob_11[(0) + 1]), "r"((unsigned int)(live_1)) : "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_342 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_342), "f"(tot_1471) : "memory"); + uint32_t _addr_374 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_374), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4936,9 +5175,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_8 = offset_1488 + _tmem_load_25[0]; if (t_33 < clen) { { - auto _gst_343 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_343)); - *(reinterpret_cast(_gst_343) + (0)) = dg_8; + auto _gst_375 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_375)); + *(reinterpret_cast(_gst_375) + (0)) = dg_8; } } int t_34 = 16 + 8 * th + 1; @@ -4946,9 +5185,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_9 = offset_1488 + _tmem_load_25[1]; if (t_34 < clen) { { - auto _gst_344 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_344)); - *(reinterpret_cast(_gst_344) + (0)) = dg_9; + auto _gst_376 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_376)); + *(reinterpret_cast(_gst_376) + (0)) = dg_9; } } int t_35 = 16 + 8 * th + 2; @@ -4956,9 +5195,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_10 = offset_1488 + _tmem_load_25[2]; if (t_35 < clen) { { - auto _gst_345 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_10; + auto _gst_377 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_10; } } int t_36 = 16 + 8 * th + 3; @@ -4966,9 +5205,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_11 = offset_1488 + _tmem_load_25[3]; if (t_36 < clen) { { - auto _gst_346 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_11; + auto _gst_378 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_11; } } int t_37 = 16 + 8 * th + 4; @@ -4976,9 +5215,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_12 = offset_1488 + _tmem_load_25[4]; if (t_37 < clen) { { - auto _gst_347 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_12; + auto _gst_379 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_12; } } int t_38 = 16 + 8 * th + 5; @@ -4986,9 +5225,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_13 = offset_1488 + _tmem_load_25[5]; if (t_38 < clen) { { - auto _gst_348 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_13; + auto _gst_380 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_13; } } int t_39 = 16 + 8 * th + 6; @@ -4996,9 +5235,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_14 = offset_1488 + _tmem_load_25[6]; if (t_39 < clen) { { - auto _gst_349 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_14; + auto _gst_381 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_14; } } int t_40 = 16 + 8 * th + 7; @@ -5006,15 +5245,15 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_15 = offset_1488 + _tmem_load_25[7]; if (t_40 < clen) { { - auto _gst_350 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_15; + auto _gst_382 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_1486 + t_lo_1487; { - uint32_t _addr_351 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_351), "f"(off_acc[0]) : "memory"); + uint32_t _addr_383 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_383), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -5068,7 +5307,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[7]; - float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, (dk2b_17 - _tmem_load_28[7]) * k_t_17) + dgf_17; + float qe_23 = q_t_1_24 * e_dg_17; + float kp_35 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_35; + float kbe_23 = (float)kb_23 * e_dg_17; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_35 * e_dg_17) * _tmem_load_27[7]); + float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, __fmaf_rn(dk2b_17 - _tmem_load_28[7], k_t_17, cr_24)) + dgf_17; s_run_1524[0] = s_run_1524[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5100,7 +5344,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[6]; - float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, (dk2b_18 - _tmem_load_28[6]) * k_t_18) + dgf_18; + float qe_24 = q_t_1_25 * e_dg_18; + float kp_36 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_36; + float kbe_24 = (float)kb_24 * e_dg_18; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_36 * e_dg_18) * _tmem_load_27[6]); + float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, __fmaf_rn(dk2b_18 - _tmem_load_28[6], k_t_18, cr_25)) + dgf_18; s_run_1524[0] = s_run_1524[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5132,7 +5381,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[5]; - float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, (dk2b_19 - _tmem_load_28[5]) * k_t_19) + dgf_19; + float qe_25 = q_t_1_26 * e_dg_19; + float kp_37 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_37; + float kbe_25 = (float)kb_25 * e_dg_19; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_37 * e_dg_19) * _tmem_load_27[5]); + float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, __fmaf_rn(dk2b_19 - _tmem_load_28[5], k_t_19, cr_26)) + dgf_19; s_run_1524[0] = s_run_1524[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5164,7 +5418,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[4]; - float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, (dk2b_20 - _tmem_load_28[4]) * k_t_20) + dgf_20; + float qe_26 = q_t_1_27 * e_dg_20; + float kp_38 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_38; + float kbe_26 = (float)kb_26 * e_dg_20; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_38 * e_dg_20) * _tmem_load_27[4]); + float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, __fmaf_rn(dk2b_20 - _tmem_load_28[4], k_t_20, cr_27)) + dgf_20; s_run_1524[0] = s_run_1524[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5196,7 +5455,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[3]; - float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, (dk2b_21 - _tmem_load_28[3]) * k_t_21) + dgf_21; + float qe_27 = q_t_1_28 * e_dg_21; + float kp_39 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_39; + float kbe_27 = (float)kb_27 * e_dg_21; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_39 * e_dg_21) * _tmem_load_27[3]); + float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, __fmaf_rn(dk2b_21 - _tmem_load_28[3], k_t_21, cr_28)) + dgf_21; s_run_1524[0] = s_run_1524[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5228,7 +5492,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[2]; - float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, (dk2b_22 - _tmem_load_28[2]) * k_t_22) + dgf_22; + float qe_28 = q_t_1_29 * e_dg_22; + float kp_40 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_40; + float kbe_28 = (float)kb_28 * e_dg_22; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_40 * e_dg_22) * _tmem_load_27[2]); + float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, __fmaf_rn(dk2b_22 - _tmem_load_28[2], k_t_22, cr_29)) + dgf_22; s_run_1524[0] = s_run_1524[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5260,7 +5529,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[1]; - float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, (dk2b_23 - _tmem_load_28[1]) * k_t_23) + dgf_23; + float qe_29 = q_t_1_30 * e_dg_23; + float kp_41 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_41; + float kbe_29 = (float)kb_29 * e_dg_23; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_41 * e_dg_23) * _tmem_load_27[1]); + float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, __fmaf_rn(dk2b_23 - _tmem_load_28[1], k_t_23, cr_30)) + dgf_23; s_run_1524[0] = s_run_1524[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5292,7 +5566,12 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + _tmem_load_28[0]; - float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, (dk2b_24 - _tmem_load_28[0]) * k_t_24) + dgf_24; + float qe_30 = q_t_1_31 * e_dg_24; + float kp_42 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_42; + float kbe_30 = (float)kb_30 * e_dg_24; + float cr_31 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_42 * e_dg_24) * _tmem_load_27[0]); + float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, __fmaf_rn(dk2b_24 - _tmem_load_28[0], k_t_24, cr_31)) + dgf_24; s_run_1524[0] = s_run_1524[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5307,31 +5586,31 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; float send_13 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_352 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_352; + float _shfl_xor_sync_384 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_384; float send_14 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_353 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_353; + float _shfl_xor_sync_385 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_385; float send_15 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_354 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_354; + float _shfl_xor_sync_386 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_386; float send_16 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_355 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_355; + float _shfl_xor_sync_387 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_387; float v2_1702[2]; float send_17 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_356 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_356; + float _shfl_xor_sync_388 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_388; float send_18 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_357 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_357; + float _shfl_xor_sync_389 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_389; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_358 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_358; - float _shfl_xor_sync_359 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_359; - float _shfl_xor_sync_360 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_360; + float _shfl_xor_sync_390 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_390; + float _shfl_xor_sync_391 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_391; + float _shfl_xor_sync_392 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_392; int ew_3 = 98304 - 32768 * th >> 2; float nz_2 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); float2 nz2_2 = make_float2(nz_2, nz_2); @@ -5341,8 +5620,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM int qq_2 = lane_1 >> 1 & 3; int j_2 = qd + 4 * tb_2; bool live_2 = 8 * th + j_2 < clen; - int _min_361 = ((8 * th + j_2) < (clen - 1) ? (8 * th + j_2) : (clen - 1)); - long long row_2 = ext0 + (long long)_min_361; + int _min_393 = ((8 * th + j_2) < (clen - 1) ? (8 * th + j_2) : (clen - 1)); + long long row_2 = ext0 + (long long)_min_393; bool is_k_2 = tk_2 == 1; __nv_bfloat16* xr_2 = ((is_k_2) ? (k_raw) : (q_raw)); int xts_2 = ((is_k_2) ? k_raw_row_stride : q_raw_row_stride); @@ -5368,43 +5647,43 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM : "=r"(*reinterpret_cast(&g_2[12])), "=r"(*reinterpret_cast(&g_2[(12) + 1])), "=r"(*reinterpret_cast(&g_2[(12) + 2])), "=r"(*reinterpret_cast(&g_2[(12) + 3])) : "r"(pool_f32_addr + (unsigned int)(4 * (sb_2 + 8 * (3 ^ qq_2))))); { - uint2 _vld_362; - _vld_362 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (0 ^ qq_2))) + 0); - uint32_t* _vpairs_362 = reinterpret_cast(&_vld_362); + uint2 _vld_394; + _vld_394 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (0 ^ qq_2))) + 0); + uint32_t* _vpairs_394 = reinterpret_cast(&_vld_394); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_362[_pair]) << 16); - (&xv_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_362[_pair]) & 0xffff0000u); + (&xv_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_394[_pair]) << 16); + (&xv_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_394[_pair]) & 0xffff0000u); } } { - uint2 _vld_363; - _vld_363 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (1 ^ qq_2))) + 0); - uint32_t* _vpairs_363 = reinterpret_cast(&_vld_363); + uint2 _vld_395; + _vld_395 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (1 ^ qq_2))) + 0); + uint32_t* _vpairs_395 = reinterpret_cast(&_vld_395); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_2[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_363[_pair]) << 16); - (&xv_2[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_363[_pair]) & 0xffff0000u); + (&xv_2[4 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_395[_pair]) << 16); + (&xv_2[4 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_395[_pair]) & 0xffff0000u); } } { - uint2 _vld_364; - _vld_364 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (2 ^ qq_2))) + 0); - uint32_t* _vpairs_364 = reinterpret_cast(&_vld_364); + uint2 _vld_396; + _vld_396 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (2 ^ qq_2))) + 0); + uint32_t* _vpairs_396 = reinterpret_cast(&_vld_396); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_2[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_364[_pair]) << 16); - (&xv_2[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_364[_pair]) & 0xffff0000u); + (&xv_2[8 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_396[_pair]) << 16); + (&xv_2[8 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_396[_pair]) & 0xffff0000u); } } { - uint2 _vld_365; - _vld_365 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (3 ^ qq_2))) + 0); - uint32_t* _vpairs_365 = reinterpret_cast(&_vld_365); + uint2 _vld_397; + _vld_397 = *reinterpret_cast(xr_2 + (xrow_2 + (long long)(8 * (3 ^ qq_2))) + 0); + uint32_t* _vpairs_397 = reinterpret_cast(&_vld_397); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xv_2[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_365[_pair]) << 16); - (&xv_2[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_365[_pair]) & 0xffff0000u); + (&xv_2[12 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_397[_pair]) << 16); + (&xv_2[12 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_397[_pair]) & 0xffff0000u); } } { @@ -5413,65 +5692,65 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 r2_2 = make_float2(rv_2[0], rv_2[0]); float y_2[16]; float p_2[16]; - float2 _f32x2_mul_a_366 = make_float2(xv_2[0], xv_2[1]); + float2 _f32x2_mul_a_398 = make_float2(xv_2[0], xv_2[1]); float2 y2_24; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_24) : "l"(*(const unsigned long long*)&_f32x2_mul_a_366), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_24) : "l"(*(const unsigned long long*)&_f32x2_mul_a_398), "l"(*(const unsigned long long*)&r2_2)); y_2[0] = y2_24.x; y_2[1] = y2_24.y; float2 p2_37 = fma_f32x2_rn_noftz(make_float2(g_2[0], g_2[1]), y2_24, nz2_2); p_2[0] = p2_37.x; p_2[1] = p2_37.y; - float2 _f32x2_mul_a_367 = make_float2(xv_2[2], xv_2[3]); + float2 _f32x2_mul_a_399 = make_float2(xv_2[2], xv_2[3]); float2 y2_25; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_25) : "l"(*(const unsigned long long*)&_f32x2_mul_a_367), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_25) : "l"(*(const unsigned long long*)&_f32x2_mul_a_399), "l"(*(const unsigned long long*)&r2_2)); y_2[2] = y2_25.x; y_2[3] = y2_25.y; float2 p2_38 = fma_f32x2_rn_noftz(make_float2(g_2[2], g_2[3]), y2_25, nz2_2); p_2[2] = p2_38.x; p_2[3] = p2_38.y; - float2 _f32x2_mul_a_368 = make_float2(xv_2[4], xv_2[5]); + float2 _f32x2_mul_a_400 = make_float2(xv_2[4], xv_2[5]); float2 y2_26; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_26) : "l"(*(const unsigned long long*)&_f32x2_mul_a_368), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_26) : "l"(*(const unsigned long long*)&_f32x2_mul_a_400), "l"(*(const unsigned long long*)&r2_2)); y_2[4] = y2_26.x; y_2[5] = y2_26.y; float2 p2_39 = fma_f32x2_rn_noftz(make_float2(g_2[4], g_2[5]), y2_26, nz2_2); p_2[4] = p2_39.x; p_2[5] = p2_39.y; - float2 _f32x2_mul_a_369 = make_float2(xv_2[6], xv_2[7]); + float2 _f32x2_mul_a_401 = make_float2(xv_2[6], xv_2[7]); float2 y2_27; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_27) : "l"(*(const unsigned long long*)&_f32x2_mul_a_369), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_27) : "l"(*(const unsigned long long*)&_f32x2_mul_a_401), "l"(*(const unsigned long long*)&r2_2)); y_2[6] = y2_27.x; y_2[7] = y2_27.y; float2 p2_40 = fma_f32x2_rn_noftz(make_float2(g_2[6], g_2[7]), y2_27, nz2_2); p_2[6] = p2_40.x; p_2[7] = p2_40.y; - float2 _f32x2_mul_a_370 = make_float2(xv_2[8], xv_2[9]); + float2 _f32x2_mul_a_402 = make_float2(xv_2[8], xv_2[9]); float2 y2_28; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_28) : "l"(*(const unsigned long long*)&_f32x2_mul_a_370), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_28) : "l"(*(const unsigned long long*)&_f32x2_mul_a_402), "l"(*(const unsigned long long*)&r2_2)); y_2[8] = y2_28.x; y_2[9] = y2_28.y; float2 p2_41 = fma_f32x2_rn_noftz(make_float2(g_2[8], g_2[9]), y2_28, nz2_2); p_2[8] = p2_41.x; p_2[9] = p2_41.y; - float2 _f32x2_mul_a_371 = make_float2(xv_2[10], xv_2[11]); + float2 _f32x2_mul_a_403 = make_float2(xv_2[10], xv_2[11]); float2 y2_29; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_29) : "l"(*(const unsigned long long*)&_f32x2_mul_a_371), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_29) : "l"(*(const unsigned long long*)&_f32x2_mul_a_403), "l"(*(const unsigned long long*)&r2_2)); y_2[10] = y2_29.x; y_2[11] = y2_29.y; float2 p2_42 = fma_f32x2_rn_noftz(make_float2(g_2[10], g_2[11]), y2_29, nz2_2); p_2[10] = p2_42.x; p_2[11] = p2_42.y; - float2 _f32x2_mul_a_372 = make_float2(xv_2[12], xv_2[13]); + float2 _f32x2_mul_a_404 = make_float2(xv_2[12], xv_2[13]); float2 y2_30; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_30) : "l"(*(const unsigned long long*)&_f32x2_mul_a_372), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_30) : "l"(*(const unsigned long long*)&_f32x2_mul_a_404), "l"(*(const unsigned long long*)&r2_2)); y_2[12] = y2_30.x; y_2[13] = y2_30.y; float2 p2_43 = fma_f32x2_rn_noftz(make_float2(g_2[12], g_2[13]), y2_30, nz2_2); p_2[12] = p2_43.x; p_2[13] = p2_43.y; - float2 _f32x2_mul_a_373 = make_float2(xv_2[14], xv_2[15]); + float2 _f32x2_mul_a_405 = make_float2(xv_2[14], xv_2[15]); float2 y2_31; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_31) : "l"(*(const unsigned long long*)&_f32x2_mul_a_373), "l"(*(const unsigned long long*)&r2_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&y2_31) : "l"(*(const unsigned long long*)&_f32x2_mul_a_405), "l"(*(const unsigned long long*)&r2_2)); y_2[14] = y2_31.x; y_2[15] = y2_31.y; float2 p2_44 = fma_f32x2_rn_noftz(make_float2(g_2[14], g_2[15]), y2_31, nz2_2); @@ -5488,35 +5767,35 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float2 e2_14 = add_f32x2_noftz(a2_7, c2_7); e_2[2] = e2_14.x; e_2[3] = e2_14.y; - float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, e_2[0], 1); - float _add_374; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_374) : "f"(e_2[0]), "f"(_shfl_xor_sync_375)); - e_2[0] = _add_374; - float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, e_2[1], 1); - float _add_376; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_376) : "f"(e_2[1]), "f"(_shfl_xor_sync_377)); - e_2[1] = _add_376; - float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, e_2[2], 1); - float _add_378; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_378) : "f"(e_2[2]), "f"(_shfl_xor_sync_379)); - e_2[2] = _add_378; - float _shfl_xor_sync_381 = __shfl_xor_sync(0xFFFFFFFF, e_2[3], 1); - float _add_380; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_380) : "f"(e_2[3]), "f"(_shfl_xor_sync_381)); - e_2[3] = _add_380; + float _shfl_xor_sync_407 = __shfl_xor_sync(0xFFFFFFFF, e_2[0], 1); + float _add_406; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_406) : "f"(e_2[0]), "f"(_shfl_xor_sync_407)); + e_2[0] = _add_406; + float _shfl_xor_sync_409 = __shfl_xor_sync(0xFFFFFFFF, e_2[1], 1); + float _add_408; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_408) : "f"(e_2[1]), "f"(_shfl_xor_sync_409)); + e_2[1] = _add_408; + float _shfl_xor_sync_411 = __shfl_xor_sync(0xFFFFFFFF, e_2[2], 1); + float _add_410; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_410) : "f"(e_2[2]), "f"(_shfl_xor_sync_411)); + e_2[2] = _add_410; + float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, e_2[3], 1); + float _add_412; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_412) : "f"(e_2[3]), "f"(_shfl_xor_sync_413)); + e_2[3] = _add_412; float2 f2_2 = add_f32x2_noftz(make_float2(e_2[0], e_2[1]), make_float2(e_2[2], e_2[3])); float sq_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2) : "f"(f2_2.x), "f"(f2_2.y)); - float _shfl_sync_384 = __shfl_sync(0xFFFFFFFF, sq_2, 0, 8); - float _shfl_sync_385 = __shfl_sync(0xFFFFFFFF, sq_2, 2, 8); - float _add_383; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_383) : "f"(_shfl_sync_384), "f"(_shfl_sync_385)); - float _shfl_sync_386 = __shfl_sync(0xFFFFFFFF, sq_2, 4, 8); - float _add_382; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_382) : "f"(_add_383), "f"(_shfl_sync_386)); - float _shfl_sync_387 = __shfl_sync(0xFFFFFFFF, sq_2, 6, 8); + float _shfl_sync_416 = __shfl_sync(0xFFFFFFFF, sq_2, 0, 8); + float _shfl_sync_417 = __shfl_sync(0xFFFFFFFF, sq_2, 2, 8); + float _add_415; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_415) : "f"(_shfl_sync_416), "f"(_shfl_sync_417)); + float _shfl_sync_418 = __shfl_sync(0xFFFFFFFF, sq_2, 4, 8); + float _add_414; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_414) : "f"(_add_415), "f"(_shfl_sync_418)); + float _shfl_sync_419 = __shfl_sync(0xFFFFFFFF, sq_2, 6, 8); float dot_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_382), "f"(_shfl_sync_387)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_414), "f"(_shfl_sync_419)); float2 d2_63 = make_float2(dot_2, dot_2); float o_12[4]; float2 t2_24 = fma_f32x2_rn_noftz(make_float2(y_2[0], y_2[1]), d2_63, nz2_2); @@ -5600,8 +5879,8 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM asm volatile("{ .reg .pred p_; setp.ne.b32 p_, %3, 0; @p_ st.global.v2.b32 [%0], {%1, %2}; }" :: "l"(dxp_2 + (orow_2 + (long long)(8 * (3 ^ qq_2)))), "r"(ob_15[(0) + 0]), "r"(ob_15[(0) + 1]), "r"((unsigned int)(live_2)) : "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_388 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_388), "f"(tot_1693) : "memory"); + uint32_t _addr_420 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_420), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5630,9 +5909,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_16 = offset_1710 + _tmem_load_29[0]; if (t_49 < clen) { { - auto _gst_389 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_389)); - *(reinterpret_cast(_gst_389) + (0)) = dg_16; + auto _gst_421 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_421)); + *(reinterpret_cast(_gst_421) + (0)) = dg_16; } } int t_50 = 8 * th + 1; @@ -5640,9 +5919,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_17 = offset_1710 + _tmem_load_29[1]; if (t_50 < clen) { { - auto _gst_390 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_390)); - *(reinterpret_cast(_gst_390) + (0)) = dg_17; + auto _gst_422 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_422)); + *(reinterpret_cast(_gst_422) + (0)) = dg_17; } } int t_51 = 8 * th + 2; @@ -5650,9 +5929,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_18 = offset_1710 + _tmem_load_29[2]; if (t_51 < clen) { { - auto _gst_391 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_391)); - *(reinterpret_cast(_gst_391) + (0)) = dg_18; + auto _gst_423 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_423)); + *(reinterpret_cast(_gst_423) + (0)) = dg_18; } } int t_52 = 8 * th + 3; @@ -5660,9 +5939,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_19 = offset_1710 + _tmem_load_29[3]; if (t_52 < clen) { { - auto _gst_392 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_392)); - *(reinterpret_cast(_gst_392) + (0)) = dg_19; + auto _gst_424 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_424)); + *(reinterpret_cast(_gst_424) + (0)) = dg_19; } } int t_53 = 8 * th + 4; @@ -5670,9 +5949,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_20 = offset_1710 + _tmem_load_29[4]; if (t_53 < clen) { { - auto _gst_393 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_393)); - *(reinterpret_cast(_gst_393) + (0)) = dg_20; + auto _gst_425 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_425)); + *(reinterpret_cast(_gst_425) + (0)) = dg_20; } } int t_54 = 8 * th + 5; @@ -5680,9 +5959,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_21 = offset_1710 + _tmem_load_29[5]; if (t_54 < clen) { { - auto _gst_394 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_394)); - *(reinterpret_cast(_gst_394) + (0)) = dg_21; + auto _gst_426 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_426)); + *(reinterpret_cast(_gst_426) + (0)) = dg_21; } } int t_55 = 8 * th + 6; @@ -5690,9 +5969,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_22 = offset_1710 + _tmem_load_29[6]; if (t_55 < clen) { { - auto _gst_395 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_395)); - *(reinterpret_cast(_gst_395) + (0)) = dg_22; + auto _gst_427 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_427)); + *(reinterpret_cast(_gst_427) + (0)) = dg_22; } } int t_56 = 8 * th + 7; @@ -5700,9 +5979,9 @@ kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorM float dg_23 = offset_1710 + _tmem_load_29[7]; if (t_56 < clen) { { - auto _gst_396 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_396)); - *(reinterpret_cast(_gst_396) + (0)) = dg_23; + auto _gst_428 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_428)); + *(reinterpret_cast(_gst_428) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1708 + t_lo_1709; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu index 1ccc6d124b9..c37ebe2a51c 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_aa6b5c23c97e1cb9880a_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2249,12 +2272,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_3 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * e2_3); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_3, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_3 : __fmaf_rn(dl, _rcp_170, e2_3)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2278,12 +2307,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_4 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * e2_4); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_4, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_4, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_4 : __fmaf_rn(dl_1, _rcp_174, e2_4)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2307,12 +2342,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_5 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * e2_5); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_5, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_5, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_5 : __fmaf_rn(dl_3, _rcp_178, e2_5)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2336,12 +2377,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_6 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * e2_6); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_6, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_6, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_6 : __fmaf_rn(dl_5, _rcp_182, e2_6)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_2 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_1 = approx_exp2(dc2_2.x); float e2_1_1 = approx_exp2(dc2_2.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_190 = approx_rcp(k_t0_2); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_191 = approx_rcp(k_t1_2); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_2.x; pan_641[3] = p2_2.y; int t0_3 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_2 = approx_exp2(dc2_3.x); float e2_1_2 = approx_exp2(dc2_3.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_194 = approx_rcp(k_t0_3); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_195 = approx_rcp(k_t1_3); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_3.x; pan_641[5] = p2_3.y; int t0_4 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(_f2_61, make_float2(g_t0_4, g_t1_4)); float e2_0_3 = approx_exp2(dc2_4.x); float e2_1_3 = approx_exp2(dc2_4.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_198 = approx_rcp(k_t0_4); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_199 = approx_rcp(k_t1_4); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_4.x; pan_641[7] = p2_4.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2505,12 +2576,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_202 = approx_rcp(k_t0_5); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_203 = approx_rcp(k_t1_5); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_5.x; pan_703[1] = p2_5.y; int t0_6 = 8 * th + 2; @@ -2534,12 +2611,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_2.x); float en_1_1 = approx_exp2(dn2_2.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_206 = approx_rcp(k_t0_6); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_207 = approx_rcp(k_t1_6); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_6.x; pan_703[3] = p2_6.y; int t0_7 = 8 * th + 4; @@ -2563,12 +2646,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_3.x); float en_1_2 = approx_exp2(dn2_3.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_210 = approx_rcp(k_t0_7); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_211 = approx_rcp(k_t1_7); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_7.x; pan_703[5] = p2_7.y; int t0_8 = 8 * th + 6; @@ -2592,12 +2681,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_4.x); float en_1_3 = approx_exp2(dn2_4.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_214 = approx_rcp(k_t0_8); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_215 = approx_rcp(k_t1_8); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_8.x; pan_703[7] = p2_8.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2639,12 +2734,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_5.x); float en_1_4 = approx_exp2(dn2_5.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_218 = approx_rcp(k_t0_9); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_219 = approx_rcp(k_t1_9); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_9.x; pan_777[1] = p2_9.y; int t0_10 = 16 + 8 * th + 2; @@ -2668,12 +2769,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_6.x); float en_1_5 = approx_exp2(dn2_6.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_222 = approx_rcp(k_t0_10); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_223 = approx_rcp(k_t1_10); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_10.x; pan_777[3] = p2_10.y; int t0_11 = 16 + 8 * th + 4; @@ -2697,12 +2804,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_7.x); float en_1_6 = approx_exp2(dn2_7.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_226 = approx_rcp(k_t0_11); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_227 = approx_rcp(k_t1_11); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_11.x; pan_777[5] = p2_11.y; int t0_12 = 16 + 8 * th + 6; @@ -2726,12 +2839,18 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 dn2_8 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_8.x); float en_1_7 = approx_exp2(dn2_8.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_230 = approx_rcp(k_t0_12); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_231 = approx_rcp(k_t1_12); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_12.x; pan_777[7] = p2_12.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2802,37 +2921,37 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_4 = approx_exp2(d2_57.x); float e1_4 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_4, e1_4); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_4, e1_4); float2 v2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_4.x; v_562_0[1] = v2_4.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_5 = approx_exp2(d2_58.x); float e1_5 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_5, e1_5); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_5, e1_5); float2 v2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_5.x; v_562_0[3] = v2_5.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_6 = approx_exp2(d2_59.x); float e1_6 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_6, e1_6); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_6, e1_6); float2 v2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_6.x; v_562_0[5] = v2_6.y; float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_7 = approx_exp2(d2_60.x); float e1_7 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_7, e1_7); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_7, e1_7); float2 v2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_7.x; v_562_0[7] = v2_7.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2966,7 +3085,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -2976,14 +3100,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_0 = t_1_0 * qrs; if (t_1_0 < clen) { { - auto _gst_216 = dq_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_1_0; + auto _gst_248 = dq_base + qoff_1_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_1_0; } { - auto _gst_217 = dk_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_1_0; + auto _gst_249 = dk_base + qoff_1_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_1_0; } } int t_1_1 = 32 + 8 * th + 6; @@ -3009,7 +3133,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3019,14 +3148,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_1 = t_1_1 * qrs; if (t_1_1 < clen) { { - auto _gst_218 = dq_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1_1; + auto _gst_250 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1_1; } { - auto _gst_219 = dk_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1_1; + auto _gst_251 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1_1; } } int t_1_2 = 32 + 8 * th + 5; @@ -3052,7 +3181,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_2)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3062,14 +3196,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_2 = t_1_2 * qrs; if (t_1_2 < clen) { { - auto _gst_220 = dq_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1_2; + auto _gst_252 = dq_base + qoff_1_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_1_2; } { - auto _gst_221 = dk_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1_2; + auto _gst_253 = dk_base + qoff_1_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_1_2; } } int t_1_3 = 32 + 8 * th + 4; @@ -3095,7 +3229,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_3)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3105,14 +3244,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_3 = t_1_3 * qrs; if (t_1_3 < clen) { { - auto _gst_222 = dq_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_1_3; + auto _gst_254 = dq_base + qoff_1_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_1_3; } { - auto _gst_223 = dk_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_1_3; + auto _gst_255 = dk_base + qoff_1_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_1_3; } } int t_1_4 = 32 + 8 * th + 3; @@ -3138,7 +3277,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_4)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3148,14 +3292,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_4 = t_1_4 * qrs; if (t_1_4 < clen) { { - auto _gst_224 = dq_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_1_4; + auto _gst_256 = dq_base + qoff_1_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_1_4; } { - auto _gst_225 = dk_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_1_4; + auto _gst_257 = dk_base + qoff_1_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_1_4; } } int t_1_5 = 32 + 8 * th + 2; @@ -3181,7 +3325,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_5)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3191,14 +3340,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_5 = t_1_5 * qrs; if (t_1_5 < clen) { { - auto _gst_226 = dq_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_1_5; + auto _gst_258 = dq_base + qoff_1_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_1_5; } { - auto _gst_227 = dk_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_1_5; + auto _gst_259 = dk_base + qoff_1_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_1_5; } } int t_1_6 = 32 + 8 * th + 1; @@ -3224,7 +3373,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_6)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3234,14 +3388,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_6 = t_1_6 * qrs; if (t_1_6 < clen) { { - auto _gst_228 = dq_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_1_6; + auto _gst_260 = dq_base + qoff_1_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_1_6; } { - auto _gst_229 = dk_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_1_6; + auto _gst_261 = dk_base + qoff_1_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_1_6; } } int t_1_7 = 32 + 8 * th; @@ -3267,7 +3421,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_7)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3277,14 +3436,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_1_7 = t_1_7 * qrs; if (t_1_7 < clen) { { - auto _gst_230 = dq_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_1_7; + auto _gst_262 = dq_base + qoff_1_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_1_7; } { - auto _gst_231 = dk_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_1_7; + auto _gst_263 = dk_base + qoff_1_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_1_7; } } float tot = s_run[0]; @@ -3293,31 +3452,31 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3377,7 +3536,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; + float qe_7 = q_t_1_8 * e_dg_0; + float kp_19 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_0; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_14[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_0) * _tmem_load_15[7]); + float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_16[7], k_t_0, cr_8)) + dgf_0; s_run_1059[0] = s_run_1059[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3387,14 +3551,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_241 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_273 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_242 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_274 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_2 = 48 + 8 * th + 6; @@ -3420,7 +3584,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[6]; - float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, (dk2b_2 - _tmem_load_16[6]) * k_t_2) + dgf_2; + float qe_8 = q_t_1_9 * e_dg_2; + float kp_20 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_14[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_2) * _tmem_load_15[6]); + float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_16[6], k_t_2, cr_9)) + dgf_2; s_run_1059[0] = s_run_1059[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3430,14 +3599,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_243 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_275 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_244 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_276 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 48 + 8 * th + 5; @@ -3463,7 +3632,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[5]; - float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, (dk2b_3 - _tmem_load_16[5]) * k_t_3) + dgf_3; + float qe_9 = q_t_1_10 * e_dg_3; + float kp_21 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_3; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_14[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_3) * _tmem_load_15[5]); + float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_16[5], k_t_3, cr_10)) + dgf_3; s_run_1059[0] = s_run_1059[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3473,14 +3647,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_245 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_277 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_246 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_278 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 48 + 8 * th + 4; @@ -3506,7 +3680,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[4]; - float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, (dk2b_4 - _tmem_load_16[4]) * k_t_4) + dgf_4; + float qe_10 = q_t_1_11 * e_dg_4; + float kp_22 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_4; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_14[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_4) * _tmem_load_15[4]); + float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_16[4], k_t_4, cr_11)) + dgf_4; s_run_1059[0] = s_run_1059[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3516,14 +3695,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_247 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_279 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_248 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_280 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 48 + 8 * th + 3; @@ -3549,7 +3728,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[3]; - float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, (dk2b_5 - _tmem_load_16[3]) * k_t_5) + dgf_5; + float qe_11 = q_t_1_12 * e_dg_5; + float kp_23 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_5; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_14[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_5) * _tmem_load_15[3]); + float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_16[3], k_t_5, cr_12)) + dgf_5; s_run_1059[0] = s_run_1059[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3559,14 +3743,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_249 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_281 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_250 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_282 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 48 + 8 * th + 2; @@ -3592,7 +3776,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[2]; - float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, (dk2b_6 - _tmem_load_16[2]) * k_t_6) + dgf_6; + float qe_12 = q_t_1_13 * e_dg_6; + float kp_24 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_6; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_14[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_6) * _tmem_load_15[2]); + float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_16[2], k_t_6, cr_13)) + dgf_6; s_run_1059[0] = s_run_1059[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3602,14 +3791,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_251 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_283 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_252 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_284 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 48 + 8 * th + 1; @@ -3635,7 +3824,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[1]; - float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, (dk2b_7 - _tmem_load_16[1]) * k_t_7) + dgf_7; + float qe_13 = q_t_1_14 * e_dg_7; + float kp_25 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_7; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_14[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_7) * _tmem_load_15[1]); + float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_16[1], k_t_7, cr_14)) + dgf_7; s_run_1059[0] = s_run_1059[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3645,14 +3839,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_253 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_285 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_254 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_286 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_7; } } int t_8 = 48 + 8 * th; @@ -3678,7 +3872,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_16[0]; - float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, (dk2b_8 - _tmem_load_16[0]) * k_t_8) + dgf_8; + float qe_14 = q_t_1_15 * e_dg_8; + float kp_26 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_8; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_14[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_8) * _tmem_load_15[0]); + float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_16[0], k_t_8, cr_15)) + dgf_8; s_run_1059[0] = s_run_1059[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3688,14 +3887,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_255 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_287 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_256 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_288 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_8; } } float tot_1228 = s_run_1059[0]; @@ -3704,36 +3903,36 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int b2_1231 = lane_1 >> 2 & 1; float v4_1232[4]; float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_289; float send_2 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_290; float send_3 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_291; float send_4 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_292; float v2_1237[2]; float send_5 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_293; float send_6 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); + v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_294; float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); + float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); + v1_1241 = v1_1241 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); + v1_1241 = v1_1241 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_1228) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_1228) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3762,9 +3961,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_0 = offset + _tmem_load_17[0]; if (t_9 < clen) { { - auto _gst_267 = dg_base + goff_1_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_1_0; + auto _gst_299 = dg_base + goff_1_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_1_0; } } int t_10 = 48 + 8 * th + 1; @@ -3772,9 +3971,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_1 = offset + _tmem_load_17[1]; if (t_10 < clen) { { - auto _gst_268 = dg_base + goff_1_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1_1; + auto _gst_300 = dg_base + goff_1_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1_1; } } int t_11 = 48 + 8 * th + 2; @@ -3782,9 +3981,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_2 = offset + _tmem_load_17[2]; if (t_11 < clen) { { - auto _gst_269 = dg_base + goff_1_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_1_2; + auto _gst_301 = dg_base + goff_1_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_1_2; } } int t_12 = 48 + 8 * th + 3; @@ -3792,9 +3991,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_3 = offset + _tmem_load_17[3]; if (t_12 < clen) { { - auto _gst_270 = dg_base + goff_1_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_1_3; + auto _gst_302 = dg_base + goff_1_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_1_3; } } int t_13 = 48 + 8 * th + 4; @@ -3802,9 +4001,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_4 = offset + _tmem_load_17[4]; if (t_13 < clen) { { - auto _gst_271 = dg_base + goff_1_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_1_4; + auto _gst_303 = dg_base + goff_1_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_1_4; } } int t_14 = 48 + 8 * th + 5; @@ -3812,9 +4011,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_5 = offset + _tmem_load_17[5]; if (t_14 < clen) { { - auto _gst_272 = dg_base + goff_1_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_1_5; + auto _gst_304 = dg_base + goff_1_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_1_5; } } int t_15 = 48 + 8 * th + 6; @@ -3822,9 +4021,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_6 = offset + _tmem_load_17[6]; if (t_15 < clen) { { - auto _gst_273 = dg_base + goff_1_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_1_6; + auto _gst_305 = dg_base + goff_1_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_1_6; } } int t_16 = 48 + 8 * th + 7; @@ -3832,9 +4031,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1_7 = offset + _tmem_load_17[7]; if (t_16 < clen) { { - auto _gst_274 = dg_base + goff_1_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_1_7; + auto _gst_306 = dg_base + goff_1_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_1_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3852,9 +4051,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_0 = offset_1250 + _tmem_load_19[0]; if (t_17 < clen) { { - auto _gst_275 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_0; + auto _gst_307 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_0; } } int t_18 = 32 + 8 * th + 1; @@ -3862,9 +4061,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_1 = offset_1250 + _tmem_load_19[1]; if (t_18 < clen) { { - auto _gst_276 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_1; + auto _gst_308 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_1; } } int t_19 = 32 + 8 * th + 2; @@ -3872,9 +4071,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_2 = offset_1250 + _tmem_load_19[2]; if (t_19 < clen) { { - auto _gst_277 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_2; + auto _gst_309 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_2; } } int t_20 = 32 + 8 * th + 3; @@ -3882,9 +4081,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_3 = offset_1250 + _tmem_load_19[3]; if (t_20 < clen) { { - auto _gst_278 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_3; + auto _gst_310 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_3; } } int t_21 = 32 + 8 * th + 4; @@ -3892,9 +4091,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_4 = offset_1250 + _tmem_load_19[4]; if (t_21 < clen) { { - auto _gst_279 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_4; + auto _gst_311 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_4; } } int t_22 = 32 + 8 * th + 5; @@ -3902,9 +4101,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_5 = offset_1250 + _tmem_load_19[5]; if (t_22 < clen) { { - auto _gst_280 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_5; + auto _gst_312 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_5; } } int t_23 = 32 + 8 * th + 6; @@ -3912,9 +4111,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_6 = offset_1250 + _tmem_load_19[6]; if (t_23 < clen) { { - auto _gst_281 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_6; + auto _gst_313 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_6; } } int t_24 = 32 + 8 * th + 7; @@ -3922,15 +4121,15 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_7 = offset_1250 + _tmem_load_19[7]; if (t_24 < clen) { { - auto _gst_282 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_7; + auto _gst_314 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi_1248 + t_lo_1249; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3990,7 +4189,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[7]; - float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, (dk2b_9 - _tmem_load_24[7]) * k_t_9) + dgf_9; + float qe_15 = q_t_1_16 * e_dg_9; + float kp_27 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_9; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_9) * _tmem_load_23[7]); + float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_24[7], k_t_9, cr_16)) + dgf_9; s_run_1302[0] = s_run_1302[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4000,14 +4204,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_9 = t_25 * qrs; if (t_25 < clen) { { - auto _gst_284 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_316 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_285 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_317 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_26 = 16 + 8 * th + 6; @@ -4033,7 +4237,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[6]; - float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, (dk2b_10 - _tmem_load_24[6]) * k_t_10) + dgf_10; + float qe_16 = q_t_1_17 * e_dg_10; + float kp_28 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_10; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_10) * _tmem_load_23[6]); + float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_24[6], k_t_10, cr_17)) + dgf_10; s_run_1302[0] = s_run_1302[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4043,14 +4252,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_10 = t_26 * qrs; if (t_26 < clen) { { - auto _gst_286 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_318 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_287 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_319 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_27 = 16 + 8 * th + 5; @@ -4076,7 +4285,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[5]; - float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, (dk2b_11 - _tmem_load_24[5]) * k_t_11) + dgf_11; + float qe_17 = q_t_1_18 * e_dg_11; + float kp_29 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_11; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_11) * _tmem_load_23[5]); + float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_24[5], k_t_11, cr_18)) + dgf_11; s_run_1302[0] = s_run_1302[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4086,14 +4300,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_11 = t_27 * qrs; if (t_27 < clen) { { - auto _gst_288 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_320 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_289 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_321 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_28 = 16 + 8 * th + 4; @@ -4119,7 +4333,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[4]; - float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, (dk2b_12 - _tmem_load_24[4]) * k_t_12) + dgf_12; + float qe_18 = q_t_1_19 * e_dg_12; + float kp_30 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_12; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_12) * _tmem_load_23[4]); + float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_24[4], k_t_12, cr_19)) + dgf_12; s_run_1302[0] = s_run_1302[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4129,14 +4348,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_12 = t_28 * qrs; if (t_28 < clen) { { - auto _gst_290 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_322 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_291 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_323 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_29 = 16 + 8 * th + 3; @@ -4162,7 +4381,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[3]; - float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, (dk2b_13 - _tmem_load_24[3]) * k_t_13) + dgf_13; + float qe_19 = q_t_1_20 * e_dg_13; + float kp_31 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_13; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_13) * _tmem_load_23[3]); + float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_24[3], k_t_13, cr_20)) + dgf_13; s_run_1302[0] = s_run_1302[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4172,14 +4396,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_13 = t_29 * qrs; if (t_29 < clen) { { - auto _gst_292 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_324 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_293 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_325 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_30 = 16 + 8 * th + 2; @@ -4205,7 +4429,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[2]; - float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, (dk2b_14 - _tmem_load_24[2]) * k_t_14) + dgf_14; + float qe_20 = q_t_1_21 * e_dg_14; + float kp_32 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_14; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_14) * _tmem_load_23[2]); + float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_24[2], k_t_14, cr_21)) + dgf_14; s_run_1302[0] = s_run_1302[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4215,14 +4444,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_14 = t_30 * qrs; if (t_30 < clen) { { - auto _gst_294 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_326 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_295 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_327 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_31 = 16 + 8 * th + 1; @@ -4248,7 +4477,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[1]; - float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, (dk2b_15 - _tmem_load_24[1]) * k_t_15) + dgf_15; + float qe_21 = q_t_1_22 * e_dg_15; + float kp_33 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_15; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_15) * _tmem_load_23[1]); + float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_24[1], k_t_15, cr_22)) + dgf_15; s_run_1302[0] = s_run_1302[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4258,14 +4492,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_15 = t_31 * qrs; if (t_31 < clen) { { - auto _gst_296 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_328 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_297 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_329 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_15; } } int t_32 = 16 + 8 * th; @@ -4291,7 +4525,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_24[0]; - float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, (dk2b_16 - _tmem_load_24[0]) * k_t_16) + dgf_16; + float qe_22 = q_t_1_23 * e_dg_16; + float kp_34 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_16; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_16) * _tmem_load_23[0]); + float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, __fmaf_rn(dk2b_16 - _tmem_load_24[0], k_t_16, cr_23)) + dgf_16; s_run_1302[0] = s_run_1302[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4301,14 +4540,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_298 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_330 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_299 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_331 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_16; } } float tot_1471 = s_run_1302[0]; @@ -4317,37 +4556,37 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; float send_7 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_332; float send_8 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_333; float send_9 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_334; float send_10 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_335; float v2_1480[2]; float send_11 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_336; float send_12 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_337; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1471) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4376,9 +4615,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_8 = offset_1488 + _tmem_load_25[0]; if (t_33 < clen) { { - auto _gst_310 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_8; + auto _gst_342 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_8; } } int t_34 = 16 + 8 * th + 1; @@ -4386,9 +4625,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_9 = offset_1488 + _tmem_load_25[1]; if (t_34 < clen) { { - auto _gst_311 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_9; + auto _gst_343 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_9; } } int t_35 = 16 + 8 * th + 2; @@ -4396,9 +4635,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_10 = offset_1488 + _tmem_load_25[2]; if (t_35 < clen) { { - auto _gst_312 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_10; + auto _gst_344 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_10; } } int t_36 = 16 + 8 * th + 3; @@ -4406,9 +4645,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_11 = offset_1488 + _tmem_load_25[3]; if (t_36 < clen) { { - auto _gst_313 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_11; + auto _gst_345 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_11; } } int t_37 = 16 + 8 * th + 4; @@ -4416,9 +4655,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_12 = offset_1488 + _tmem_load_25[4]; if (t_37 < clen) { { - auto _gst_314 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_12; + auto _gst_346 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_12; } } int t_38 = 16 + 8 * th + 5; @@ -4426,9 +4665,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_13 = offset_1488 + _tmem_load_25[5]; if (t_38 < clen) { { - auto _gst_315 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_13; + auto _gst_347 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_13; } } int t_39 = 16 + 8 * th + 6; @@ -4436,9 +4675,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_14 = offset_1488 + _tmem_load_25[6]; if (t_39 < clen) { { - auto _gst_316 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_14; + auto _gst_348 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_14; } } int t_40 = 16 + 8 * th + 7; @@ -4446,15 +4685,15 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_15 = offset_1488 + _tmem_load_25[7]; if (t_40 < clen) { { - auto _gst_317 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_15; + auto _gst_349 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_1486 + t_lo_1487; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4508,7 +4747,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[7]; - float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, (dk2b_17 - _tmem_load_28[7]) * k_t_17) + dgf_17; + float qe_23 = q_t_1_24 * e_dg_17; + float kp_35 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_35; + float kbe_23 = (float)kb_23 * e_dg_17; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_35 * e_dg_17) * _tmem_load_27[7]); + float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, __fmaf_rn(dk2b_17 - _tmem_load_28[7], k_t_17, cr_24)) + dgf_17; s_run_1524[0] = s_run_1524[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4518,14 +4762,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_17 = t_41 * qrs; if (t_41 < clen) { { - auto _gst_319 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_351 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_320 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_352 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_42 = 8 * th + 6; @@ -4551,7 +4795,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[6]; - float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, (dk2b_18 - _tmem_load_28[6]) * k_t_18) + dgf_18; + float qe_24 = q_t_1_25 * e_dg_18; + float kp_36 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_36; + float kbe_24 = (float)kb_24 * e_dg_18; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_36 * e_dg_18) * _tmem_load_27[6]); + float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, __fmaf_rn(dk2b_18 - _tmem_load_28[6], k_t_18, cr_25)) + dgf_18; s_run_1524[0] = s_run_1524[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4561,14 +4810,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_18 = t_42 * qrs; if (t_42 < clen) { { - auto _gst_321 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_353 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_322 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_354 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_43 = 8 * th + 5; @@ -4594,7 +4843,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[5]; - float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, (dk2b_19 - _tmem_load_28[5]) * k_t_19) + dgf_19; + float qe_25 = q_t_1_26 * e_dg_19; + float kp_37 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_37; + float kbe_25 = (float)kb_25 * e_dg_19; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_37 * e_dg_19) * _tmem_load_27[5]); + float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, __fmaf_rn(dk2b_19 - _tmem_load_28[5], k_t_19, cr_26)) + dgf_19; s_run_1524[0] = s_run_1524[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4604,14 +4858,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_19 = t_43 * qrs; if (t_43 < clen) { { - auto _gst_323 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_355 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_324 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_356 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_44 = 8 * th + 4; @@ -4637,7 +4891,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[4]; - float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, (dk2b_20 - _tmem_load_28[4]) * k_t_20) + dgf_20; + float qe_26 = q_t_1_27 * e_dg_20; + float kp_38 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_38; + float kbe_26 = (float)kb_26 * e_dg_20; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_38 * e_dg_20) * _tmem_load_27[4]); + float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, __fmaf_rn(dk2b_20 - _tmem_load_28[4], k_t_20, cr_27)) + dgf_20; s_run_1524[0] = s_run_1524[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4647,14 +4906,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_20 = t_44 * qrs; if (t_44 < clen) { { - auto _gst_325 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_357 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_326 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_358 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_45 = 8 * th + 3; @@ -4680,7 +4939,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[3]; - float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, (dk2b_21 - _tmem_load_28[3]) * k_t_21) + dgf_21; + float qe_27 = q_t_1_28 * e_dg_21; + float kp_39 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_39; + float kbe_27 = (float)kb_27 * e_dg_21; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_39 * e_dg_21) * _tmem_load_27[3]); + float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, __fmaf_rn(dk2b_21 - _tmem_load_28[3], k_t_21, cr_28)) + dgf_21; s_run_1524[0] = s_run_1524[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4690,14 +4954,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_21 = t_45 * qrs; if (t_45 < clen) { { - auto _gst_327 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_359 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_328 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_360 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_46 = 8 * th + 2; @@ -4723,7 +4987,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[2]; - float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, (dk2b_22 - _tmem_load_28[2]) * k_t_22) + dgf_22; + float qe_28 = q_t_1_29 * e_dg_22; + float kp_40 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_40; + float kbe_28 = (float)kb_28 * e_dg_22; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_40 * e_dg_22) * _tmem_load_27[2]); + float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, __fmaf_rn(dk2b_22 - _tmem_load_28[2], k_t_22, cr_29)) + dgf_22; s_run_1524[0] = s_run_1524[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4733,14 +5002,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_22 = t_46 * qrs; if (t_46 < clen) { { - auto _gst_329 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_361 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_330 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_362 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_47 = 8 * th + 1; @@ -4766,7 +5035,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[1]; - float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, (dk2b_23 - _tmem_load_28[1]) * k_t_23) + dgf_23; + float qe_29 = q_t_1_30 * e_dg_23; + float kp_41 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_41; + float kbe_29 = (float)kb_29 * e_dg_23; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_41 * e_dg_23) * _tmem_load_27[1]); + float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, __fmaf_rn(dk2b_23 - _tmem_load_28[1], k_t_23, cr_30)) + dgf_23; s_run_1524[0] = s_run_1524[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4776,14 +5050,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_23 = t_47 * qrs; if (t_47 < clen) { { - auto _gst_331 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_363 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_332 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_364 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_23; } } int t_48 = 8 * th; @@ -4809,7 +5083,12 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + _tmem_load_28[0]; - float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, (dk2b_24 - _tmem_load_28[0]) * k_t_24) + dgf_24; + float qe_30 = q_t_1_31 * e_dg_24; + float kp_42 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_42; + float kbe_30 = (float)kb_30 * e_dg_24; + float cr_31 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_42 * e_dg_24) * _tmem_load_27[0]); + float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, __fmaf_rn(dk2b_24 - _tmem_load_28[0], k_t_24, cr_31)) + dgf_24; s_run_1524[0] = s_run_1524[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4819,14 +5098,14 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_333 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_365 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_334 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_366 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_24; } } float tot_1693 = s_run_1524[0]; @@ -4835,35 +5114,35 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; float send_13 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_367; float send_14 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_368; float send_15 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_369; float send_16 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_370; float v2_1702[2]; float send_17 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_371; float send_18 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_372; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_1693) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4892,9 +5171,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_16 = offset_1710 + _tmem_load_29[0]; if (t_49 < clen) { { - auto _gst_345 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_16; + auto _gst_377 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_16; } } int t_50 = 8 * th + 1; @@ -4902,9 +5181,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_17 = offset_1710 + _tmem_load_29[1]; if (t_50 < clen) { { - auto _gst_346 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_17; + auto _gst_378 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_17; } } int t_51 = 8 * th + 2; @@ -4912,9 +5191,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_18 = offset_1710 + _tmem_load_29[2]; if (t_51 < clen) { { - auto _gst_347 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_18; + auto _gst_379 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_18; } } int t_52 = 8 * th + 3; @@ -4922,9 +5201,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_19 = offset_1710 + _tmem_load_29[3]; if (t_52 < clen) { { - auto _gst_348 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_19; + auto _gst_380 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_19; } } int t_53 = 8 * th + 4; @@ -4932,9 +5211,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_20 = offset_1710 + _tmem_load_29[4]; if (t_53 < clen) { { - auto _gst_349 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_20; + auto _gst_381 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_20; } } int t_54 = 8 * th + 5; @@ -4942,9 +5221,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_21 = offset_1710 + _tmem_load_29[5]; if (t_54 < clen) { { - auto _gst_350 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_21; + auto _gst_382 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_21; } } int t_55 = 8 * th + 6; @@ -4952,9 +5231,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_22 = offset_1710 + _tmem_load_29[6]; if (t_55 < clen) { { - auto _gst_351 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_22; + auto _gst_383 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_22; } } int t_56 = 8 * th + 7; @@ -4962,9 +5241,9 @@ kernel_cake_kda_det_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorM float dg_23 = offset_1710 + _tmem_load_29[7]; if (t_56 < clen) { { - auto _gst_352 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_23; + auto _gst_384 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1708 + t_lo_1709; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_binding.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu similarity index 98% rename from flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_binding.cu rename to flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu index 680c8258f80..111665227c8 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_9ad80f98f22e0b3bce08_binding.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_kda_det_train_9ad80f98f22e0b3bce08, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_kda_det_train_ad4c93dbb0435933c8a1, for the device source that defines it. Do not edit. #include #include #include @@ -23,13 +23,13 @@ #include -extern "C" __global__ void kernel_cake_kda_det_train_9ad80f98f22e0b3bce08(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride); +extern "C" __global__ void kernel_cake_kda_det_train_ad4c93dbb0435933c8a1(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride); -namespace cake_binding_kernel_cake_kda_det_train_9ad80f98f22e0b3bce08 { +namespace cake_binding_kernel_cake_kda_det_train_ad4c93dbb0435933c8a1 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_kda_det_train_9ad80f98f22e0b3bce08"; +constexpr const char* KERNEL_NAME = "kernel_cake_kda_det_train_ad4c93dbb0435933c8a1"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -487,7 +487,7 @@ void Run(TensorView a_dAqk, TensorView a_dAkk, TensorView a_gk, TensorView a_k_e const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { CheckCuda(cudaSetDevice(dev.device_id), "cudaSetDevice"); - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_kda_det_train_9ad80f98f22e0b3bce08), cudaFuncAttributeMaxDynamicSharedMemorySize, 115200), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_kda_det_train_ad4c93dbb0435933c8a1), cudaFuncAttributeMaxDynamicSharedMemorySize, 115200), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -527,11 +527,11 @@ void Run(TensorView a_dAqk, TensorView a_dAkk, TensorView a_gk, TensorView a_k_e config.blockDim = dim3(288, 1, 1); config.dynamicSmemBytes = 115200; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_kda_det_train_9ad80f98f22e0b3bce08), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_kda_det_train_ad4c93dbb0435933c8a1), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_kda_det_train_9ad80f98f22e0b3bce08 +} // namespace cake_binding_kernel_cake_kda_det_train_ad4c93dbb0435933c8a1 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_kda_det_train_9ad80f98f22e0b3bce08::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_kda_det_train_ad4c93dbb0435933c8a1::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu similarity index 85% rename from flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu rename to flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu index 63210c6849e..652b450a3bb 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_328afe43951979e798d1_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ad4c93dbb0435933c8a1_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -182,6 +189,21 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { } +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + __device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { float r; asm("mul.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); @@ -191,7 +213,7 @@ __device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void -kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride) +kernel_cake_kda_det_train_ad4c93dbb0435933c8a1(const __grid_constant__ CUtensorMap dAqk, const __grid_constant__ CUtensorMap dAkk, const __grid_constant__ CUtensorMap gk, const __grid_constant__ CUtensorMap k_e, const __grid_constant__ CUtensorMap q_e, const __grid_constant__ CUtensorMap dq_f_t, const __grid_constant__ CUtensorMap dk_f_t, const __grid_constant__ CUtensorMap dg_f_t, float* __restrict__ beta, float* __restrict__ dq_f, float* __restrict__ dk_f, float* __restrict__ dg_f, float* __restrict__ db_f, int* __restrict__ chunk_bos, int* __restrict__ chunk_len, __nv_bfloat16* __restrict__ q_raw, __nv_bfloat16* __restrict__ k_raw, float* __restrict__ qk_rstd, __nv_bfloat16* __restrict__ dq_ext, __nv_bfloat16* __restrict__ dk_ext, float* __restrict__ dg_ext, __nv_bfloat16* __restrict__ dbeta_ext, int num_heads, int num_qk_heads, int group, int num_tokens, int q_raw_row_stride, int q_raw_head_stride, int k_raw_row_stride, int k_raw_head_stride) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -2256,12 +2278,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_3 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * e2_3); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_3, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_3 : __fmaf_rn(dl, _rcp_170, e2_3)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2285,12 +2313,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_4 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * e2_4); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_4, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_4, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_4 : __fmaf_rn(dl_1, _rcp_174, e2_4)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2314,12 +2348,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_5 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * e2_5); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_5, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_5, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_5 : __fmaf_rn(dl_3, _rcp_178, e2_5)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2343,12 +2383,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_6 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * e2_6); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_6, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_6, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_6 : __fmaf_rn(dl_5, _rcp_182, e2_6)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2384,12 +2430,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_2 = 48 + 8 * th + 2; @@ -2409,12 +2461,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_1 = approx_exp2(dc2_2.x); float e2_1_1 = approx_exp2(dc2_2.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_190 = approx_rcp(k_t0_2); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_191 = approx_rcp(k_t1_2); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_2.x; pan_641[3] = p2_2.y; int t0_3 = 48 + 8 * th + 4; @@ -2434,12 +2492,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_2 = approx_exp2(dc2_3.x); float e2_1_2 = approx_exp2(dc2_3.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_194 = approx_rcp(k_t0_3); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_195 = approx_rcp(k_t1_3); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_3.x; pan_641[5] = p2_3.y; int t0_4 = 48 + 8 * th + 6; @@ -2459,12 +2523,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(_f2_61, make_float2(g_t0_4, g_t1_4)); float e2_0_3 = approx_exp2(dc2_4.x); float e2_1_3 = approx_exp2(dc2_4.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_198 = approx_rcp(k_t0_4); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_199 = approx_rcp(k_t1_4); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_4.x; pan_641[7] = p2_4.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2512,12 +2582,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_202 = approx_rcp(k_t0_5); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_203 = approx_rcp(k_t1_5); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_5.x; pan_703[1] = p2_5.y; int t0_6 = 8 * th + 2; @@ -2541,12 +2617,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_2.x); float en_1_1 = approx_exp2(dn2_2.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_206 = approx_rcp(k_t0_6); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_207 = approx_rcp(k_t1_6); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_6.x; pan_703[3] = p2_6.y; int t0_7 = 8 * th + 4; @@ -2570,12 +2652,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_3.x); float en_1_2 = approx_exp2(dn2_3.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_210 = approx_rcp(k_t0_7); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_211 = approx_rcp(k_t1_7); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_7.x; pan_703[5] = p2_7.y; int t0_8 = 8 * th + 6; @@ -2599,12 +2687,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_4.x); float en_1_3 = approx_exp2(dn2_4.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_214 = approx_rcp(k_t0_8); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_215 = approx_rcp(k_t1_8); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_8.x; pan_703[7] = p2_8.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2646,12 +2740,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_5.x); float en_1_4 = approx_exp2(dn2_5.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_218 = approx_rcp(k_t0_9); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_219 = approx_rcp(k_t1_9); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_9.x; pan_777[1] = p2_9.y; int t0_10 = 16 + 8 * th + 2; @@ -2675,12 +2775,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_6.x); float en_1_5 = approx_exp2(dn2_6.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_222 = approx_rcp(k_t0_10); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_223 = approx_rcp(k_t1_10); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_10.x; pan_777[3] = p2_10.y; int t0_11 = 16 + 8 * th + 4; @@ -2704,12 +2810,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_7.x); float en_1_6 = approx_exp2(dn2_7.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_226 = approx_rcp(k_t0_11); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_227 = approx_rcp(k_t1_11); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_11.x; pan_777[5] = p2_11.y; int t0_12 = 16 + 8 * th + 6; @@ -2733,12 +2845,18 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 dn2_8 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_8.x); float en_1_7 = approx_exp2(dn2_8.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_230 = approx_rcp(k_t0_12); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_231 = approx_rcp(k_t1_12); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_12.x; pan_777[7] = p2_12.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2809,37 +2927,37 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_4 = approx_exp2(d2_57.x); float e1_4 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_4, e1_4); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_4, e1_4); float2 v2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_4.x; v_562_0[1] = v2_4.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_5 = approx_exp2(d2_58.x); float e1_5 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_5, e1_5); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_5, e1_5); float2 v2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_5.x; v_562_0[3] = v2_5.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_6 = approx_exp2(d2_59.x); float e1_6 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_6, e1_6); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_6, e1_6); float2 v2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_6.x; v_562_0[5] = v2_6.y; float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_7 = approx_exp2(d2_60.x); float e1_7 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_7, e1_7); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_7, e1_7); float2 v2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_7.x; v_562_0[7] = v2_7.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2851,29 +2969,29 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2881,29 +2999,29 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2973,7 +3091,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3005,7 +3128,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3037,7 +3165,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_2)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3069,7 +3202,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_3)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3101,7 +3239,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_4)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3133,7 +3276,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_5)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3165,7 +3313,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_6)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3197,7 +3350,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_7)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3212,41 +3370,41 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_216 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_216; + float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_248; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_217 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_217; + float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_249; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_218 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_218; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_250; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_219 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_219; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_251; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_220 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_220; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_252; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_221 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_221; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_253; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_222 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_222; - float _shfl_xor_sync_223 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_223; - float _shfl_xor_sync_224 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_224; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_254; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_255; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_256; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); int ew_0 = 98304 - 32768 * th >> 2; float nz = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); bool live0 = 32 + 8 * th + qd < clen; bool live1 = 32 + 8 * th + qd + 4 < clen; - int _min_225 = ((32 + 8 * th + qd) < (clen - 1) ? (32 + 8 * th + qd) : (clen - 1)); - long long rowa = ext0 + (long long)_min_225; - int _min_226 = ((32 + 8 * th + qd + 4) < (clen - 1) ? (32 + 8 * th + qd + 4) : (clen - 1)); - long long rowb = ext0 + (long long)_min_226; + int _min_257 = ((32 + 8 * th + qd) < (clen - 1) ? (32 + 8 * th + qd) : (clen - 1)); + long long rowa = ext0 + (long long)_min_257; + int _min_258 = ((32 + 8 * th + qd + 4) < (clen - 1) ? (32 + 8 * th + qd + 4) : (clen - 1)); + long long rowb = ext0 + (long long)_min_258; float xqa[4]; float xka[4]; float xqb[4]; @@ -3274,52 +3432,52 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (ew_0 + 1024 + 128 * (qd + 4) + 4 * lane_1)))); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint2 _vld_227; - _vld_227 = *reinterpret_cast(q_raw + (rowa * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_227 = reinterpret_cast(&_vld_227); + uint2 _vld_259; + _vld_259 = *reinterpret_cast(q_raw + (rowa * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_259 = reinterpret_cast(&_vld_259); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqa[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_227[_pair]) << 16); - (&xqa[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_227[_pair]) & 0xffff0000u); + (&xqa[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_259[_pair]) << 16); + (&xqa[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_259[_pair]) & 0xffff0000u); } } { rqa[0] = *reinterpret_cast(qk_rstd + ((0 + rowa) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_228; - _vld_228 = *reinterpret_cast(k_raw + (rowa * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_228 = reinterpret_cast(&_vld_228); + uint2 _vld_260; + _vld_260 = *reinterpret_cast(k_raw + (rowa * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_260 = reinterpret_cast(&_vld_260); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xka[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_228[_pair]) << 16); - (&xka[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_228[_pair]) & 0xffff0000u); + (&xka[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_260[_pair]) << 16); + (&xka[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_260[_pair]) & 0xffff0000u); } } { rka[0] = *reinterpret_cast(qk_rstd + (((long long)num_tokens + rowa) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_229; - _vld_229 = *reinterpret_cast(q_raw + (rowb * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_229 = reinterpret_cast(&_vld_229); + uint2 _vld_261; + _vld_261 = *reinterpret_cast(q_raw + (rowb * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_261 = reinterpret_cast(&_vld_261); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqb[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_229[_pair]) << 16); - (&xqb[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_229[_pair]) & 0xffff0000u); + (&xqb[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_261[_pair]) << 16); + (&xqb[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_261[_pair]) & 0xffff0000u); } } { rqb[0] = *reinterpret_cast(qk_rstd + ((0 + rowb) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_230; - _vld_230 = *reinterpret_cast(k_raw + (rowb * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_230 = reinterpret_cast(&_vld_230); + uint2 _vld_262; + _vld_262 = *reinterpret_cast(k_raw + (rowb * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_262 = reinterpret_cast(&_vld_262); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xkb[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_230[_pair]) << 16); - (&xkb[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_230[_pair]) & 0xffff0000u); + (&xkb[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_262[_pair]) << 16); + (&xkb[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_262[_pair]) & 0xffff0000u); } } { @@ -3359,106 +3517,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y[4]; float p[4]; y[0] = mul_f32_rn_noftz(xq[0], rq[0]); - float _fma_231; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_231) : "f"(gq[0]), "f"(y[0]), "f"(nz)); - p[0] = _fma_231; + float _fma_263; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_263) : "f"(gq[0]), "f"(y[0]), "f"(nz)); + p[0] = _fma_263; y[1] = mul_f32_rn_noftz(xq[1], rq[0]); - float _fma_232; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_232) : "f"(gq[1]), "f"(y[1]), "f"(nz)); - p[1] = _fma_232; + float _fma_264; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_264) : "f"(gq[1]), "f"(y[1]), "f"(nz)); + p[1] = _fma_264; y[2] = mul_f32_rn_noftz(xq[2], rq[0]); - float _fma_233; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_233) : "f"(gq[2]), "f"(y[2]), "f"(nz)); - p[2] = _fma_233; + float _fma_265; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_265) : "f"(gq[2]), "f"(y[2]), "f"(nz)); + p[2] = _fma_265; y[3] = mul_f32_rn_noftz(xq[3], rq[0]); - float _fma_234; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_234) : "f"(gq[3]), "f"(y[3]), "f"(nz)); - p[3] = _fma_234; - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, p[0], 4); - float _add_235; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_235) : "f"(p[0]), "f"(_shfl_xor_sync_236)); - p[0] = _add_235; - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, p[1], 4); - float _add_237; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_237) : "f"(p[1]), "f"(_shfl_xor_sync_238)); - p[1] = _add_237; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, p[2], 4); - float _add_239; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_239) : "f"(p[2]), "f"(_shfl_xor_sync_240)); - p[2] = _add_239; - float _shfl_xor_sync_242 = __shfl_xor_sync(0xFFFFFFFF, p[3], 4); - float _add_241; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_241) : "f"(p[3]), "f"(_shfl_xor_sync_242)); - p[3] = _add_241; - float _shfl_xor_sync_244 = __shfl_xor_sync(0xFFFFFFFF, p[0], 2); - float _add_243; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_243) : "f"(p[0]), "f"(_shfl_xor_sync_244)); - p[0] = _add_243; - float _shfl_xor_sync_246 = __shfl_xor_sync(0xFFFFFFFF, p[1], 2); - float _add_245; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_245) : "f"(p[1]), "f"(_shfl_xor_sync_246)); - p[1] = _add_245; - float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, p[2], 2); - float _add_247; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_247) : "f"(p[2]), "f"(_shfl_xor_sync_248)); - p[2] = _add_247; - float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, p[3], 2); - float _add_249; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_249) : "f"(p[3]), "f"(_shfl_xor_sync_250)); - p[3] = _add_249; - float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, p[0], 1); - float _add_251; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_251) : "f"(p[0]), "f"(_shfl_xor_sync_252)); - p[0] = _add_251; - float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, p[1], 1); - float _add_253; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_253) : "f"(p[1]), "f"(_shfl_xor_sync_254)); - p[1] = _add_253; - float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, p[2], 1); - float _add_255; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_255) : "f"(p[2]), "f"(_shfl_xor_sync_256)); - p[2] = _add_255; - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, p[3], 1); - float _add_257; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_257) : "f"(p[3]), "f"(_shfl_xor_sync_258)); - p[3] = _add_257; - float _add_259; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_259) : "f"(p[0]), "f"(p[2])); - float _add_260; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_260) : "f"(p[1]), "f"(p[3])); + float _fma_266; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_266) : "f"(gq[3]), "f"(y[3]), "f"(nz)); + p[3] = _fma_266; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, p[0], 4); + float _add_267; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_267) : "f"(p[0]), "f"(_shfl_xor_sync_268)); + p[0] = _add_267; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, p[1], 4); + float _add_269; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_269) : "f"(p[1]), "f"(_shfl_xor_sync_270)); + p[1] = _add_269; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, p[2], 4); + float _add_271; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_271) : "f"(p[2]), "f"(_shfl_xor_sync_272)); + p[2] = _add_271; + float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, p[3], 4); + float _add_273; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_273) : "f"(p[3]), "f"(_shfl_xor_sync_274)); + p[3] = _add_273; + float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, p[0], 2); + float _add_275; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_275) : "f"(p[0]), "f"(_shfl_xor_sync_276)); + p[0] = _add_275; + float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, p[1], 2); + float _add_277; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_277) : "f"(p[1]), "f"(_shfl_xor_sync_278)); + p[1] = _add_277; + float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, p[2], 2); + float _add_279; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_279) : "f"(p[2]), "f"(_shfl_xor_sync_280)); + p[2] = _add_279; + float _shfl_xor_sync_282 = __shfl_xor_sync(0xFFFFFFFF, p[3], 2); + float _add_281; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_281) : "f"(p[3]), "f"(_shfl_xor_sync_282)); + p[3] = _add_281; + float _shfl_xor_sync_284 = __shfl_xor_sync(0xFFFFFFFF, p[0], 1); + float _add_283; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_283) : "f"(p[0]), "f"(_shfl_xor_sync_284)); + p[0] = _add_283; + float _shfl_xor_sync_286 = __shfl_xor_sync(0xFFFFFFFF, p[1], 1); + float _add_285; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_285) : "f"(p[1]), "f"(_shfl_xor_sync_286)); + p[1] = _add_285; + float _shfl_xor_sync_288 = __shfl_xor_sync(0xFFFFFFFF, p[2], 1); + float _add_287; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_287) : "f"(p[2]), "f"(_shfl_xor_sync_288)); + p[2] = _add_287; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, p[3], 1); + float _add_289; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_289) : "f"(p[3]), "f"(_shfl_xor_sync_290)); + p[3] = _add_289; + float _add_291; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_291) : "f"(p[0]), "f"(p[2])); + float _add_292; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_292) : "f"(p[1]), "f"(p[3])); float sq; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq) : "f"(_add_259), "f"(_add_260)); - float _shfl_sync_263 = __shfl_sync(0xFFFFFFFF, sq, 0); - float _shfl_sync_264 = __shfl_sync(0xFFFFFFFF, sq, 8); - float _add_262; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_262) : "f"(_shfl_sync_263), "f"(_shfl_sync_264)); - float _shfl_sync_265 = __shfl_sync(0xFFFFFFFF, sq, 16); - float _add_261; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_261) : "f"(_add_262), "f"(_shfl_sync_265)); - float _shfl_sync_266 = __shfl_sync(0xFFFFFFFF, sq, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq) : "f"(_add_291), "f"(_add_292)); + float _shfl_sync_295 = __shfl_sync(0xFFFFFFFF, sq, 0); + float _shfl_sync_296 = __shfl_sync(0xFFFFFFFF, sq, 8); + float _add_294; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_294) : "f"(_shfl_sync_295), "f"(_shfl_sync_296)); + float _shfl_sync_297 = __shfl_sync(0xFFFFFFFF, sq, 16); + float _add_293; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_293) : "f"(_add_294), "f"(_shfl_sync_297)); + float _shfl_sync_298 = __shfl_sync(0xFFFFFFFF, sq, 24); float dot; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_261), "f"(_shfl_sync_266)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot) : "f"(_add_293), "f"(_shfl_sync_298)); float o[4]; - float _fma_268; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_268) : "f"(y[0]), "f"(dot), "f"(nz)); - float _sub_267; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_267) : "f"(gq[0]), "f"(_fma_268)); - o[0] = mul_f32_rn_noftz(rq[0], _sub_267); - float _fma_270; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_270) : "f"(y[1]), "f"(dot), "f"(nz)); - float _sub_269; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_269) : "f"(gq[1]), "f"(_fma_270)); - o[1] = mul_f32_rn_noftz(rq[0], _sub_269); - float _fma_272; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_272) : "f"(y[2]), "f"(dot), "f"(nz)); - float _sub_271; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_271) : "f"(gq[2]), "f"(_fma_272)); - o[2] = mul_f32_rn_noftz(rq[0], _sub_271); - float _fma_274; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_274) : "f"(y[3]), "f"(dot), "f"(nz)); - float _sub_273; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_273) : "f"(gq[3]), "f"(_fma_274)); - o[3] = mul_f32_rn_noftz(rq[0], _sub_273); + float _fma_300; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_300) : "f"(y[0]), "f"(dot), "f"(nz)); + float _sub_299; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_299) : "f"(gq[0]), "f"(_fma_300)); + o[0] = mul_f32_rn_noftz(rq[0], _sub_299); + float _fma_302; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_302) : "f"(y[1]), "f"(dot), "f"(nz)); + float _sub_301; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_301) : "f"(gq[1]), "f"(_fma_302)); + o[1] = mul_f32_rn_noftz(rq[0], _sub_301); + float _fma_304; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_304) : "f"(y[2]), "f"(dot), "f"(nz)); + float _sub_303; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_303) : "f"(gq[2]), "f"(_fma_304)); + o[2] = mul_f32_rn_noftz(rq[0], _sub_303); + float _fma_306; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_306) : "f"(y[3]), "f"(dot), "f"(nz)); + float _sub_305; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_305) : "f"(gq[3]), "f"(_fma_306)); + o[3] = mul_f32_rn_noftz(rq[0], _sub_305); unsigned int ob[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -3469,106 +3627,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_0[4]; float p_0[4]; y_0[0] = mul_f32_rn_noftz(xk[0], rk[0]); - float _fma_275; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_275) : "f"(gk_0[0]), "f"(y_0[0]), "f"(nz)); - p_0[0] = _fma_275; + float _fma_307; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_307) : "f"(gk_0[0]), "f"(y_0[0]), "f"(nz)); + p_0[0] = _fma_307; y_0[1] = mul_f32_rn_noftz(xk[1], rk[0]); - float _fma_276; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_276) : "f"(gk_0[1]), "f"(y_0[1]), "f"(nz)); - p_0[1] = _fma_276; + float _fma_308; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_308) : "f"(gk_0[1]), "f"(y_0[1]), "f"(nz)); + p_0[1] = _fma_308; y_0[2] = mul_f32_rn_noftz(xk[2], rk[0]); - float _fma_277; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_277) : "f"(gk_0[2]), "f"(y_0[2]), "f"(nz)); - p_0[2] = _fma_277; + float _fma_309; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_309) : "f"(gk_0[2]), "f"(y_0[2]), "f"(nz)); + p_0[2] = _fma_309; y_0[3] = mul_f32_rn_noftz(xk[3], rk[0]); - float _fma_278; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_278) : "f"(gk_0[3]), "f"(y_0[3]), "f"(nz)); - p_0[3] = _fma_278; - float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 4); - float _add_279; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_279) : "f"(p_0[0]), "f"(_shfl_xor_sync_280)); - p_0[0] = _add_279; - float _shfl_xor_sync_282 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 4); - float _add_281; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_281) : "f"(p_0[1]), "f"(_shfl_xor_sync_282)); - p_0[1] = _add_281; - float _shfl_xor_sync_284 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 4); - float _add_283; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_283) : "f"(p_0[2]), "f"(_shfl_xor_sync_284)); - p_0[2] = _add_283; - float _shfl_xor_sync_286 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 4); - float _add_285; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_285) : "f"(p_0[3]), "f"(_shfl_xor_sync_286)); - p_0[3] = _add_285; - float _shfl_xor_sync_288 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 2); - float _add_287; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_287) : "f"(p_0[0]), "f"(_shfl_xor_sync_288)); - p_0[0] = _add_287; - float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 2); - float _add_289; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_289) : "f"(p_0[1]), "f"(_shfl_xor_sync_290)); - p_0[1] = _add_289; - float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 2); - float _add_291; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_291) : "f"(p_0[2]), "f"(_shfl_xor_sync_292)); - p_0[2] = _add_291; - float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 2); - float _add_293; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_293) : "f"(p_0[3]), "f"(_shfl_xor_sync_294)); - p_0[3] = _add_293; - float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 1); - float _add_295; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_295) : "f"(p_0[0]), "f"(_shfl_xor_sync_296)); - p_0[0] = _add_295; - float _shfl_xor_sync_298 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 1); - float _add_297; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_297) : "f"(p_0[1]), "f"(_shfl_xor_sync_298)); - p_0[1] = _add_297; - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 1); - float _add_299; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_299) : "f"(p_0[2]), "f"(_shfl_xor_sync_300)); - p_0[2] = _add_299; - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 1); - float _add_301; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_301) : "f"(p_0[3]), "f"(_shfl_xor_sync_302)); - p_0[3] = _add_301; - float _add_303; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_303) : "f"(p_0[0]), "f"(p_0[2])); - float _add_304; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_304) : "f"(p_0[1]), "f"(p_0[3])); + float _fma_310; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_310) : "f"(gk_0[3]), "f"(y_0[3]), "f"(nz)); + p_0[3] = _fma_310; + float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 4); + float _add_311; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_311) : "f"(p_0[0]), "f"(_shfl_xor_sync_312)); + p_0[0] = _add_311; + float _shfl_xor_sync_314 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 4); + float _add_313; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_313) : "f"(p_0[1]), "f"(_shfl_xor_sync_314)); + p_0[1] = _add_313; + float _shfl_xor_sync_316 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 4); + float _add_315; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_315) : "f"(p_0[2]), "f"(_shfl_xor_sync_316)); + p_0[2] = _add_315; + float _shfl_xor_sync_318 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 4); + float _add_317; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_317) : "f"(p_0[3]), "f"(_shfl_xor_sync_318)); + p_0[3] = _add_317; + float _shfl_xor_sync_320 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 2); + float _add_319; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_319) : "f"(p_0[0]), "f"(_shfl_xor_sync_320)); + p_0[0] = _add_319; + float _shfl_xor_sync_322 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 2); + float _add_321; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_321) : "f"(p_0[1]), "f"(_shfl_xor_sync_322)); + p_0[1] = _add_321; + float _shfl_xor_sync_324 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 2); + float _add_323; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_323) : "f"(p_0[2]), "f"(_shfl_xor_sync_324)); + p_0[2] = _add_323; + float _shfl_xor_sync_326 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 2); + float _add_325; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_325) : "f"(p_0[3]), "f"(_shfl_xor_sync_326)); + p_0[3] = _add_325; + float _shfl_xor_sync_328 = __shfl_xor_sync(0xFFFFFFFF, p_0[0], 1); + float _add_327; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_327) : "f"(p_0[0]), "f"(_shfl_xor_sync_328)); + p_0[0] = _add_327; + float _shfl_xor_sync_330 = __shfl_xor_sync(0xFFFFFFFF, p_0[1], 1); + float _add_329; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_329) : "f"(p_0[1]), "f"(_shfl_xor_sync_330)); + p_0[1] = _add_329; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, p_0[2], 1); + float _add_331; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_331) : "f"(p_0[2]), "f"(_shfl_xor_sync_332)); + p_0[2] = _add_331; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, p_0[3], 1); + float _add_333; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_333) : "f"(p_0[3]), "f"(_shfl_xor_sync_334)); + p_0[3] = _add_333; + float _add_335; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_335) : "f"(p_0[0]), "f"(p_0[2])); + float _add_336; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_336) : "f"(p_0[1]), "f"(p_0[3])); float sq_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_0) : "f"(_add_303), "f"(_add_304)); - float _shfl_sync_307 = __shfl_sync(0xFFFFFFFF, sq_0, 0); - float _shfl_sync_308 = __shfl_sync(0xFFFFFFFF, sq_0, 8); - float _add_306; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_306) : "f"(_shfl_sync_307), "f"(_shfl_sync_308)); - float _shfl_sync_309 = __shfl_sync(0xFFFFFFFF, sq_0, 16); - float _add_305; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_305) : "f"(_add_306), "f"(_shfl_sync_309)); - float _shfl_sync_310 = __shfl_sync(0xFFFFFFFF, sq_0, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_0) : "f"(_add_335), "f"(_add_336)); + float _shfl_sync_339 = __shfl_sync(0xFFFFFFFF, sq_0, 0); + float _shfl_sync_340 = __shfl_sync(0xFFFFFFFF, sq_0, 8); + float _add_338; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_338) : "f"(_shfl_sync_339), "f"(_shfl_sync_340)); + float _shfl_sync_341 = __shfl_sync(0xFFFFFFFF, sq_0, 16); + float _add_337; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_337) : "f"(_add_338), "f"(_shfl_sync_341)); + float _shfl_sync_342 = __shfl_sync(0xFFFFFFFF, sq_0, 24); float dot_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_305), "f"(_shfl_sync_310)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_0) : "f"(_add_337), "f"(_shfl_sync_342)); float o_0[4]; - float _fma_312; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_312) : "f"(y_0[0]), "f"(dot_0), "f"(nz)); - float _sub_311; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_311) : "f"(gk_0[0]), "f"(_fma_312)); - o_0[0] = mul_f32_rn_noftz(rk[0], _sub_311); - float _fma_314; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_314) : "f"(y_0[1]), "f"(dot_0), "f"(nz)); - float _sub_313; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_313) : "f"(gk_0[1]), "f"(_fma_314)); - o_0[1] = mul_f32_rn_noftz(rk[0], _sub_313); - float _fma_316; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_316) : "f"(y_0[2]), "f"(dot_0), "f"(nz)); - float _sub_315; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_315) : "f"(gk_0[2]), "f"(_fma_316)); - o_0[2] = mul_f32_rn_noftz(rk[0], _sub_315); - float _fma_318; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_318) : "f"(y_0[3]), "f"(dot_0), "f"(nz)); - float _sub_317; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_317) : "f"(gk_0[3]), "f"(_fma_318)); - o_0[3] = mul_f32_rn_noftz(rk[0], _sub_317); + float _fma_344; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_344) : "f"(y_0[0]), "f"(dot_0), "f"(nz)); + float _sub_343; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_343) : "f"(gk_0[0]), "f"(_fma_344)); + o_0[0] = mul_f32_rn_noftz(rk[0], _sub_343); + float _fma_346; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_346) : "f"(y_0[1]), "f"(dot_0), "f"(nz)); + float _sub_345; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_345) : "f"(gk_0[1]), "f"(_fma_346)); + o_0[1] = mul_f32_rn_noftz(rk[0], _sub_345); + float _fma_348; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_348) : "f"(y_0[2]), "f"(dot_0), "f"(nz)); + float _sub_347; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_347) : "f"(gk_0[2]), "f"(_fma_348)); + o_0[2] = mul_f32_rn_noftz(rk[0], _sub_347); + float _fma_350; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_350) : "f"(y_0[3]), "f"(dot_0), "f"(nz)); + float _sub_349; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_349) : "f"(gk_0[3]), "f"(_fma_350)); + o_0[3] = mul_f32_rn_noftz(rk[0], _sub_349); unsigned int ob_0[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -3633,7 +3791,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; + float qe_7 = q_t_1_8 * e_dg_0; + float kp_19 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_0; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_14[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_0) * _tmem_load_15[7]); + float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_16[7], k_t_0, cr_8)) + dgf_0; s_run_1059[0] = s_run_1059[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3665,7 +3828,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[6]; - float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, (dk2b_2 - _tmem_load_16[6]) * k_t_2) + dgf_2; + float qe_8 = q_t_1_9 * e_dg_2; + float kp_20 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_14[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_2) * _tmem_load_15[6]); + float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_16[6], k_t_2, cr_9)) + dgf_2; s_run_1059[0] = s_run_1059[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3697,7 +3865,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[5]; - float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, (dk2b_3 - _tmem_load_16[5]) * k_t_3) + dgf_3; + float qe_9 = q_t_1_10 * e_dg_3; + float kp_21 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_3; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_14[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_3) * _tmem_load_15[5]); + float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_16[5], k_t_3, cr_10)) + dgf_3; s_run_1059[0] = s_run_1059[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3729,7 +3902,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[4]; - float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, (dk2b_4 - _tmem_load_16[4]) * k_t_4) + dgf_4; + float qe_10 = q_t_1_11 * e_dg_4; + float kp_22 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_4; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_14[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_4) * _tmem_load_15[4]); + float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_16[4], k_t_4, cr_11)) + dgf_4; s_run_1059[0] = s_run_1059[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3761,7 +3939,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[3]; - float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, (dk2b_5 - _tmem_load_16[3]) * k_t_5) + dgf_5; + float qe_11 = q_t_1_12 * e_dg_5; + float kp_23 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_5; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_14[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_5) * _tmem_load_15[3]); + float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_16[3], k_t_5, cr_12)) + dgf_5; s_run_1059[0] = s_run_1059[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3793,7 +3976,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[2]; - float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, (dk2b_6 - _tmem_load_16[2]) * k_t_6) + dgf_6; + float qe_12 = q_t_1_13 * e_dg_6; + float kp_24 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_6; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_14[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_6) * _tmem_load_15[2]); + float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_16[2], k_t_6, cr_13)) + dgf_6; s_run_1059[0] = s_run_1059[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3825,7 +4013,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[1]; - float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, (dk2b_7 - _tmem_load_16[1]) * k_t_7) + dgf_7; + float qe_13 = q_t_1_14 * e_dg_7; + float kp_25 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_7; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_14[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_7) * _tmem_load_15[1]); + float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_16[1], k_t_7, cr_14)) + dgf_7; s_run_1059[0] = s_run_1059[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3857,7 +4050,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_16[0]; - float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, (dk2b_8 - _tmem_load_16[0]) * k_t_8) + dgf_8; + float qe_14 = q_t_1_15 * e_dg_8; + float kp_26 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_8; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_14[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_8) * _tmem_load_15[0]); + float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_16[0], k_t_8, cr_15)) + dgf_8; s_run_1059[0] = s_run_1059[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3872,40 +4070,40 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int b2_1231 = lane_1 >> 2 & 1; float v4_1232[4]; float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_319 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_319; + float _shfl_xor_sync_351 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_351; float send_2 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_320 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_320; + float _shfl_xor_sync_352 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_352; float send_3 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_321 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_321; + float _shfl_xor_sync_353 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_353; float send_4 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_322 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_322; + float _shfl_xor_sync_354 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_354; float v2_1237[2]; float send_5 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_323 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_323; + float _shfl_xor_sync_355 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_355; float send_6 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_324 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_324; + float _shfl_xor_sync_356 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); + v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_356; float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_325 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_325; - float _shfl_xor_sync_326 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_326; - float _shfl_xor_sync_327 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_327; + float _shfl_xor_sync_357 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); + float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_357; + float _shfl_xor_sync_358 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); + v1_1241 = v1_1241 + _shfl_xor_sync_358; + float _shfl_xor_sync_359 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); + v1_1241 = v1_1241 + _shfl_xor_sync_359; asm volatile("tcgen05.fence::before_thread_sync;"); int ew_1 = 98304 - 32768 * th >> 2; float nz_0 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); bool live0_0 = 48 + 8 * th + qd < clen; bool live1_0 = 48 + 8 * th + qd + 4 < clen; - int _min_328 = ((48 + 8 * th + qd) < (clen - 1) ? (48 + 8 * th + qd) : (clen - 1)); - long long rowa_0 = ext0 + (long long)_min_328; - int _min_329 = ((48 + 8 * th + qd + 4) < (clen - 1) ? (48 + 8 * th + qd + 4) : (clen - 1)); - long long rowb_0 = ext0 + (long long)_min_329; + int _min_360 = ((48 + 8 * th + qd) < (clen - 1) ? (48 + 8 * th + qd) : (clen - 1)); + long long rowa_0 = ext0 + (long long)_min_360; + int _min_361 = ((48 + 8 * th + qd + 4) < (clen - 1) ? (48 + 8 * th + qd + 4) : (clen - 1)); + long long rowb_0 = ext0 + (long long)_min_361; float xqa_0[4]; float xka_0[4]; float xqb_0[4]; @@ -3933,52 +4131,52 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (ew_1 + 1024 + 128 * (qd + 4) + 4 * lane_1)))); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint2 _vld_330; - _vld_330 = *reinterpret_cast(q_raw + (rowa_0 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_330 = reinterpret_cast(&_vld_330); + uint2 _vld_362; + _vld_362 = *reinterpret_cast(q_raw + (rowa_0 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_362 = reinterpret_cast(&_vld_362); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqa_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_330[_pair]) << 16); - (&xqa_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_330[_pair]) & 0xffff0000u); + (&xqa_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_362[_pair]) << 16); + (&xqa_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_362[_pair]) & 0xffff0000u); } } { rqa_0[0] = *reinterpret_cast(qk_rstd + ((0 + rowa_0) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_331; - _vld_331 = *reinterpret_cast(k_raw + (rowa_0 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_331 = reinterpret_cast(&_vld_331); + uint2 _vld_363; + _vld_363 = *reinterpret_cast(k_raw + (rowa_0 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_363 = reinterpret_cast(&_vld_363); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xka_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_331[_pair]) << 16); - (&xka_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_331[_pair]) & 0xffff0000u); + (&xka_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_363[_pair]) << 16); + (&xka_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_363[_pair]) & 0xffff0000u); } } { rka_0[0] = *reinterpret_cast(qk_rstd + (((long long)num_tokens + rowa_0) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_332; - _vld_332 = *reinterpret_cast(q_raw + (rowb_0 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_332 = reinterpret_cast(&_vld_332); + uint2 _vld_364; + _vld_364 = *reinterpret_cast(q_raw + (rowb_0 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_364 = reinterpret_cast(&_vld_364); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqb_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_332[_pair]) << 16); - (&xqb_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_332[_pair]) & 0xffff0000u); + (&xqb_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_364[_pair]) << 16); + (&xqb_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_364[_pair]) & 0xffff0000u); } } { rqb_0[0] = *reinterpret_cast(qk_rstd + ((0 + rowb_0) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_333; - _vld_333 = *reinterpret_cast(k_raw + (rowb_0 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_333 = reinterpret_cast(&_vld_333); + uint2 _vld_365; + _vld_365 = *reinterpret_cast(k_raw + (rowb_0 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_365 = reinterpret_cast(&_vld_365); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xkb_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_333[_pair]) << 16); - (&xkb_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_333[_pair]) & 0xffff0000u); + (&xkb_0[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_365[_pair]) << 16); + (&xkb_0[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_365[_pair]) & 0xffff0000u); } } { @@ -4018,106 +4216,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_1[4]; float p_1[4]; y_1[0] = mul_f32_rn_noftz(xq_1[0], rq_1[0]); - float _fma_334; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_334) : "f"(gq_1[0]), "f"(y_1[0]), "f"(nz_0)); - p_1[0] = _fma_334; + float _fma_366; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_366) : "f"(gq_1[0]), "f"(y_1[0]), "f"(nz_0)); + p_1[0] = _fma_366; y_1[1] = mul_f32_rn_noftz(xq_1[1], rq_1[0]); - float _fma_335; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_335) : "f"(gq_1[1]), "f"(y_1[1]), "f"(nz_0)); - p_1[1] = _fma_335; + float _fma_367; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_367) : "f"(gq_1[1]), "f"(y_1[1]), "f"(nz_0)); + p_1[1] = _fma_367; y_1[2] = mul_f32_rn_noftz(xq_1[2], rq_1[0]); - float _fma_336; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_336) : "f"(gq_1[2]), "f"(y_1[2]), "f"(nz_0)); - p_1[2] = _fma_336; + float _fma_368; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_368) : "f"(gq_1[2]), "f"(y_1[2]), "f"(nz_0)); + p_1[2] = _fma_368; y_1[3] = mul_f32_rn_noftz(xq_1[3], rq_1[0]); - float _fma_337; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_337) : "f"(gq_1[3]), "f"(y_1[3]), "f"(nz_0)); - p_1[3] = _fma_337; - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 4); - float _add_338; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_338) : "f"(p_1[0]), "f"(_shfl_xor_sync_339)); - p_1[0] = _add_338; - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 4); - float _add_340; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_340) : "f"(p_1[1]), "f"(_shfl_xor_sync_341)); - p_1[1] = _add_340; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 4); - float _add_342; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_342) : "f"(p_1[2]), "f"(_shfl_xor_sync_343)); - p_1[2] = _add_342; - float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 4); - float _add_344; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_344) : "f"(p_1[3]), "f"(_shfl_xor_sync_345)); - p_1[3] = _add_344; - float _shfl_xor_sync_347 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 2); - float _add_346; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_346) : "f"(p_1[0]), "f"(_shfl_xor_sync_347)); - p_1[0] = _add_346; - float _shfl_xor_sync_349 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 2); - float _add_348; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_348) : "f"(p_1[1]), "f"(_shfl_xor_sync_349)); - p_1[1] = _add_348; - float _shfl_xor_sync_351 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 2); - float _add_350; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_350) : "f"(p_1[2]), "f"(_shfl_xor_sync_351)); - p_1[2] = _add_350; - float _shfl_xor_sync_353 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 2); - float _add_352; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_352) : "f"(p_1[3]), "f"(_shfl_xor_sync_353)); - p_1[3] = _add_352; - float _shfl_xor_sync_355 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 1); - float _add_354; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_354) : "f"(p_1[0]), "f"(_shfl_xor_sync_355)); - p_1[0] = _add_354; - float _shfl_xor_sync_357 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 1); - float _add_356; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_356) : "f"(p_1[1]), "f"(_shfl_xor_sync_357)); - p_1[1] = _add_356; - float _shfl_xor_sync_359 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 1); - float _add_358; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_358) : "f"(p_1[2]), "f"(_shfl_xor_sync_359)); - p_1[2] = _add_358; - float _shfl_xor_sync_361 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 1); - float _add_360; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_360) : "f"(p_1[3]), "f"(_shfl_xor_sync_361)); - p_1[3] = _add_360; - float _add_362; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_362) : "f"(p_1[0]), "f"(p_1[2])); - float _add_363; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_363) : "f"(p_1[1]), "f"(p_1[3])); + float _fma_369; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_369) : "f"(gq_1[3]), "f"(y_1[3]), "f"(nz_0)); + p_1[3] = _fma_369; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 4); + float _add_370; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_370) : "f"(p_1[0]), "f"(_shfl_xor_sync_371)); + p_1[0] = _add_370; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 4); + float _add_372; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_372) : "f"(p_1[1]), "f"(_shfl_xor_sync_373)); + p_1[1] = _add_372; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 4); + float _add_374; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_374) : "f"(p_1[2]), "f"(_shfl_xor_sync_375)); + p_1[2] = _add_374; + float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 4); + float _add_376; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_376) : "f"(p_1[3]), "f"(_shfl_xor_sync_377)); + p_1[3] = _add_376; + float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 2); + float _add_378; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_378) : "f"(p_1[0]), "f"(_shfl_xor_sync_379)); + p_1[0] = _add_378; + float _shfl_xor_sync_381 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 2); + float _add_380; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_380) : "f"(p_1[1]), "f"(_shfl_xor_sync_381)); + p_1[1] = _add_380; + float _shfl_xor_sync_383 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 2); + float _add_382; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_382) : "f"(p_1[2]), "f"(_shfl_xor_sync_383)); + p_1[2] = _add_382; + float _shfl_xor_sync_385 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 2); + float _add_384; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_384) : "f"(p_1[3]), "f"(_shfl_xor_sync_385)); + p_1[3] = _add_384; + float _shfl_xor_sync_387 = __shfl_xor_sync(0xFFFFFFFF, p_1[0], 1); + float _add_386; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_386) : "f"(p_1[0]), "f"(_shfl_xor_sync_387)); + p_1[0] = _add_386; + float _shfl_xor_sync_389 = __shfl_xor_sync(0xFFFFFFFF, p_1[1], 1); + float _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(p_1[1]), "f"(_shfl_xor_sync_389)); + p_1[1] = _add_388; + float _shfl_xor_sync_391 = __shfl_xor_sync(0xFFFFFFFF, p_1[2], 1); + float _add_390; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_390) : "f"(p_1[2]), "f"(_shfl_xor_sync_391)); + p_1[2] = _add_390; + float _shfl_xor_sync_393 = __shfl_xor_sync(0xFFFFFFFF, p_1[3], 1); + float _add_392; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(p_1[3]), "f"(_shfl_xor_sync_393)); + p_1[3] = _add_392; + float _add_394; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(p_1[0]), "f"(p_1[2])); + float _add_395; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_395) : "f"(p_1[1]), "f"(p_1[3])); float sq_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1) : "f"(_add_362), "f"(_add_363)); - float _shfl_sync_366 = __shfl_sync(0xFFFFFFFF, sq_1, 0); - float _shfl_sync_367 = __shfl_sync(0xFFFFFFFF, sq_1, 8); - float _add_365; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_365) : "f"(_shfl_sync_366), "f"(_shfl_sync_367)); - float _shfl_sync_368 = __shfl_sync(0xFFFFFFFF, sq_1, 16); - float _add_364; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_364) : "f"(_add_365), "f"(_shfl_sync_368)); - float _shfl_sync_369 = __shfl_sync(0xFFFFFFFF, sq_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1) : "f"(_add_394), "f"(_add_395)); + float _shfl_sync_398 = __shfl_sync(0xFFFFFFFF, sq_1, 0); + float _shfl_sync_399 = __shfl_sync(0xFFFFFFFF, sq_1, 8); + float _add_397; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_397) : "f"(_shfl_sync_398), "f"(_shfl_sync_399)); + float _shfl_sync_400 = __shfl_sync(0xFFFFFFFF, sq_1, 16); + float _add_396; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_396) : "f"(_add_397), "f"(_shfl_sync_400)); + float _shfl_sync_401 = __shfl_sync(0xFFFFFFFF, sq_1, 24); float dot_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_364), "f"(_shfl_sync_369)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1) : "f"(_add_396), "f"(_shfl_sync_401)); float o_1[4]; - float _fma_371; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_371) : "f"(y_1[0]), "f"(dot_1), "f"(nz_0)); - float _sub_370; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_370) : "f"(gq_1[0]), "f"(_fma_371)); - o_1[0] = mul_f32_rn_noftz(rq_1[0], _sub_370); - float _fma_373; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_373) : "f"(y_1[1]), "f"(dot_1), "f"(nz_0)); - float _sub_372; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_372) : "f"(gq_1[1]), "f"(_fma_373)); - o_1[1] = mul_f32_rn_noftz(rq_1[0], _sub_372); - float _fma_375; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_375) : "f"(y_1[2]), "f"(dot_1), "f"(nz_0)); - float _sub_374; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_374) : "f"(gq_1[2]), "f"(_fma_375)); - o_1[2] = mul_f32_rn_noftz(rq_1[0], _sub_374); - float _fma_377; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_377) : "f"(y_1[3]), "f"(dot_1), "f"(nz_0)); - float _sub_376; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_376) : "f"(gq_1[3]), "f"(_fma_377)); - o_1[3] = mul_f32_rn_noftz(rq_1[0], _sub_376); + float _fma_403; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_403) : "f"(y_1[0]), "f"(dot_1), "f"(nz_0)); + float _sub_402; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_402) : "f"(gq_1[0]), "f"(_fma_403)); + o_1[0] = mul_f32_rn_noftz(rq_1[0], _sub_402); + float _fma_405; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_405) : "f"(y_1[1]), "f"(dot_1), "f"(nz_0)); + float _sub_404; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_404) : "f"(gq_1[1]), "f"(_fma_405)); + o_1[1] = mul_f32_rn_noftz(rq_1[0], _sub_404); + float _fma_407; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_407) : "f"(y_1[2]), "f"(dot_1), "f"(nz_0)); + float _sub_406; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_406) : "f"(gq_1[2]), "f"(_fma_407)); + o_1[2] = mul_f32_rn_noftz(rq_1[0], _sub_406); + float _fma_409; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_409) : "f"(y_1[3]), "f"(dot_1), "f"(nz_0)); + float _sub_408; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_408) : "f"(gq_1[3]), "f"(_fma_409)); + o_1[3] = mul_f32_rn_noftz(rq_1[0], _sub_408); unsigned int ob_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -4128,106 +4326,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_1_1[4]; float p_1_1[4]; y_1_1[0] = mul_f32_rn_noftz(xk_1[0], rk_1[0]); - float _fma_378; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_378) : "f"(gk_1[0]), "f"(y_1_1[0]), "f"(nz_0)); - p_1_1[0] = _fma_378; + float _fma_410; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_410) : "f"(gk_1[0]), "f"(y_1_1[0]), "f"(nz_0)); + p_1_1[0] = _fma_410; y_1_1[1] = mul_f32_rn_noftz(xk_1[1], rk_1[0]); - float _fma_379; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_379) : "f"(gk_1[1]), "f"(y_1_1[1]), "f"(nz_0)); - p_1_1[1] = _fma_379; + float _fma_411; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_411) : "f"(gk_1[1]), "f"(y_1_1[1]), "f"(nz_0)); + p_1_1[1] = _fma_411; y_1_1[2] = mul_f32_rn_noftz(xk_1[2], rk_1[0]); - float _fma_380; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_380) : "f"(gk_1[2]), "f"(y_1_1[2]), "f"(nz_0)); - p_1_1[2] = _fma_380; + float _fma_412; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_412) : "f"(gk_1[2]), "f"(y_1_1[2]), "f"(nz_0)); + p_1_1[2] = _fma_412; y_1_1[3] = mul_f32_rn_noftz(xk_1[3], rk_1[0]); - float _fma_381; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_381) : "f"(gk_1[3]), "f"(y_1_1[3]), "f"(nz_0)); - p_1_1[3] = _fma_381; - float _shfl_xor_sync_383 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 4); - float _add_382; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_382) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_383)); - p_1_1[0] = _add_382; - float _shfl_xor_sync_385 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 4); - float _add_384; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_384) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_385)); - p_1_1[1] = _add_384; - float _shfl_xor_sync_387 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 4); - float _add_386; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_386) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_387)); - p_1_1[2] = _add_386; - float _shfl_xor_sync_389 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 4); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_389)); - p_1_1[3] = _add_388; - float _shfl_xor_sync_391 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 2); - float _add_390; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_390) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_391)); - p_1_1[0] = _add_390; - float _shfl_xor_sync_393 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 2); - float _add_392; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_393)); - p_1_1[1] = _add_392; - float _shfl_xor_sync_395 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 2); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_395)); - p_1_1[2] = _add_394; - float _shfl_xor_sync_397 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 2); - float _add_396; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_396) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_397)); - p_1_1[3] = _add_396; - float _shfl_xor_sync_399 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 1); - float _add_398; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_399)); - p_1_1[0] = _add_398; - float _shfl_xor_sync_401 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 1); - float _add_400; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_400) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_401)); - p_1_1[1] = _add_400; - float _shfl_xor_sync_403 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 1); - float _add_402; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_402) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_403)); - p_1_1[2] = _add_402; - float _shfl_xor_sync_405 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 1); - float _add_404; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_404) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_405)); - p_1_1[3] = _add_404; - float _add_406; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_406) : "f"(p_1_1[0]), "f"(p_1_1[2])); - float _add_407; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_407) : "f"(p_1_1[1]), "f"(p_1_1[3])); + float _fma_413; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_413) : "f"(gk_1[3]), "f"(y_1_1[3]), "f"(nz_0)); + p_1_1[3] = _fma_413; + float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 4); + float _add_414; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_414) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_415)); + p_1_1[0] = _add_414; + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 4); + float _add_416; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_416) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_417)); + p_1_1[1] = _add_416; + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 4); + float _add_418; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_418) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_419)); + p_1_1[2] = _add_418; + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 4); + float _add_420; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_420) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_421)); + p_1_1[3] = _add_420; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 2); + float _add_422; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_422) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_423)); + p_1_1[0] = _add_422; + float _shfl_xor_sync_425 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 2); + float _add_424; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_424) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_425)); + p_1_1[1] = _add_424; + float _shfl_xor_sync_427 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 2); + float _add_426; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_426) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_427)); + p_1_1[2] = _add_426; + float _shfl_xor_sync_429 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 2); + float _add_428; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_428) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_429)); + p_1_1[3] = _add_428; + float _shfl_xor_sync_431 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[0], 1); + float _add_430; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_430) : "f"(p_1_1[0]), "f"(_shfl_xor_sync_431)); + p_1_1[0] = _add_430; + float _shfl_xor_sync_433 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[1], 1); + float _add_432; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_432) : "f"(p_1_1[1]), "f"(_shfl_xor_sync_433)); + p_1_1[1] = _add_432; + float _shfl_xor_sync_435 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[2], 1); + float _add_434; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_434) : "f"(p_1_1[2]), "f"(_shfl_xor_sync_435)); + p_1_1[2] = _add_434; + float _shfl_xor_sync_437 = __shfl_xor_sync(0xFFFFFFFF, p_1_1[3], 1); + float _add_436; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_436) : "f"(p_1_1[3]), "f"(_shfl_xor_sync_437)); + p_1_1[3] = _add_436; + float _add_438; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_438) : "f"(p_1_1[0]), "f"(p_1_1[2])); + float _add_439; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_439) : "f"(p_1_1[1]), "f"(p_1_1[3])); float sq_1_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1_1) : "f"(_add_406), "f"(_add_407)); - float _shfl_sync_410 = __shfl_sync(0xFFFFFFFF, sq_1_1, 0); - float _shfl_sync_411 = __shfl_sync(0xFFFFFFFF, sq_1_1, 8); - float _add_409; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_409) : "f"(_shfl_sync_410), "f"(_shfl_sync_411)); - float _shfl_sync_412 = __shfl_sync(0xFFFFFFFF, sq_1_1, 16); - float _add_408; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_408) : "f"(_add_409), "f"(_shfl_sync_412)); - float _shfl_sync_413 = __shfl_sync(0xFFFFFFFF, sq_1_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_1_1) : "f"(_add_438), "f"(_add_439)); + float _shfl_sync_442 = __shfl_sync(0xFFFFFFFF, sq_1_1, 0); + float _shfl_sync_443 = __shfl_sync(0xFFFFFFFF, sq_1_1, 8); + float _add_441; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_441) : "f"(_shfl_sync_442), "f"(_shfl_sync_443)); + float _shfl_sync_444 = __shfl_sync(0xFFFFFFFF, sq_1_1, 16); + float _add_440; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_440) : "f"(_add_441), "f"(_shfl_sync_444)); + float _shfl_sync_445 = __shfl_sync(0xFFFFFFFF, sq_1_1, 24); float dot_1_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1_1) : "f"(_add_408), "f"(_shfl_sync_413)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_1_1) : "f"(_add_440), "f"(_shfl_sync_445)); float o_1_1[4]; - float _fma_415; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_415) : "f"(y_1_1[0]), "f"(dot_1_1), "f"(nz_0)); - float _sub_414; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_414) : "f"(gk_1[0]), "f"(_fma_415)); - o_1_1[0] = mul_f32_rn_noftz(rk_1[0], _sub_414); - float _fma_417; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_417) : "f"(y_1_1[1]), "f"(dot_1_1), "f"(nz_0)); - float _sub_416; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_416) : "f"(gk_1[1]), "f"(_fma_417)); - o_1_1[1] = mul_f32_rn_noftz(rk_1[0], _sub_416); - float _fma_419; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_419) : "f"(y_1_1[2]), "f"(dot_1_1), "f"(nz_0)); - float _sub_418; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_418) : "f"(gk_1[2]), "f"(_fma_419)); - o_1_1[2] = mul_f32_rn_noftz(rk_1[0], _sub_418); - float _fma_421; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_421) : "f"(y_1_1[3]), "f"(dot_1_1), "f"(nz_0)); - float _sub_420; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_420) : "f"(gk_1[3]), "f"(_fma_421)); - o_1_1[3] = mul_f32_rn_noftz(rk_1[0], _sub_420); + float _fma_447; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_447) : "f"(y_1_1[0]), "f"(dot_1_1), "f"(nz_0)); + float _sub_446; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_446) : "f"(gk_1[0]), "f"(_fma_447)); + o_1_1[0] = mul_f32_rn_noftz(rk_1[0], _sub_446); + float _fma_449; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_449) : "f"(y_1_1[1]), "f"(dot_1_1), "f"(nz_0)); + float _sub_448; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_448) : "f"(gk_1[1]), "f"(_fma_449)); + o_1_1[1] = mul_f32_rn_noftz(rk_1[0], _sub_448); + float _fma_451; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_451) : "f"(y_1_1[2]), "f"(dot_1_1), "f"(nz_0)); + float _sub_450; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_450) : "f"(gk_1[2]), "f"(_fma_451)); + o_1_1[2] = mul_f32_rn_noftz(rk_1[0], _sub_450); + float _fma_453; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_453) : "f"(y_1_1[3]), "f"(dot_1_1), "f"(nz_0)); + float _sub_452; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_452) : "f"(gk_1[3]), "f"(_fma_453)); + o_1_1[3] = mul_f32_rn_noftz(rk_1[0], _sub_452); unsigned int ob_1_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -4237,8 +4435,8 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM asm volatile("{ .reg .pred p_; setp.ne.b32 p_, %3, 0; @p_ st.global.v2.b32 [%0], {%1, %2}; }" :: "l"(dk_ext + dx_1), "r"(ob_1_1[(0) + 0]), "r"(ob_1_1[(0) + 1]), "r"((unsigned int)(live_1)) : "memory"); } { - uint32_t _addr_422 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_422), "f"(tot_1228) : "memory"); + uint32_t _addr_454 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_454), "f"(tot_1228) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4267,9 +4465,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_0 = offset + _tmem_load_17[0]; if (t_9 < clen) { { - auto _gst_423 = dg_base + goff_1_0; - __builtin_assume(__isGlobal(_gst_423)); - *(reinterpret_cast(_gst_423) + (0)) = dg_1_0; + auto _gst_455 = dg_base + goff_1_0; + __builtin_assume(__isGlobal(_gst_455)); + *(reinterpret_cast(_gst_455) + (0)) = dg_1_0; } } int t_10 = 48 + 8 * th + 1; @@ -4277,9 +4475,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_1 = offset + _tmem_load_17[1]; if (t_10 < clen) { { - auto _gst_424 = dg_base + goff_1_1; - __builtin_assume(__isGlobal(_gst_424)); - *(reinterpret_cast(_gst_424) + (0)) = dg_1_1; + auto _gst_456 = dg_base + goff_1_1; + __builtin_assume(__isGlobal(_gst_456)); + *(reinterpret_cast(_gst_456) + (0)) = dg_1_1; } } int t_11 = 48 + 8 * th + 2; @@ -4287,9 +4485,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_2 = offset + _tmem_load_17[2]; if (t_11 < clen) { { - auto _gst_425 = dg_base + goff_1_2; - __builtin_assume(__isGlobal(_gst_425)); - *(reinterpret_cast(_gst_425) + (0)) = dg_1_2; + auto _gst_457 = dg_base + goff_1_2; + __builtin_assume(__isGlobal(_gst_457)); + *(reinterpret_cast(_gst_457) + (0)) = dg_1_2; } } int t_12 = 48 + 8 * th + 3; @@ -4297,9 +4495,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_3 = offset + _tmem_load_17[3]; if (t_12 < clen) { { - auto _gst_426 = dg_base + goff_1_3; - __builtin_assume(__isGlobal(_gst_426)); - *(reinterpret_cast(_gst_426) + (0)) = dg_1_3; + auto _gst_458 = dg_base + goff_1_3; + __builtin_assume(__isGlobal(_gst_458)); + *(reinterpret_cast(_gst_458) + (0)) = dg_1_3; } } int t_13 = 48 + 8 * th + 4; @@ -4307,9 +4505,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_4 = offset + _tmem_load_17[4]; if (t_13 < clen) { { - auto _gst_427 = dg_base + goff_1_4; - __builtin_assume(__isGlobal(_gst_427)); - *(reinterpret_cast(_gst_427) + (0)) = dg_1_4; + auto _gst_459 = dg_base + goff_1_4; + __builtin_assume(__isGlobal(_gst_459)); + *(reinterpret_cast(_gst_459) + (0)) = dg_1_4; } } int t_14 = 48 + 8 * th + 5; @@ -4317,9 +4515,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_5 = offset + _tmem_load_17[5]; if (t_14 < clen) { { - auto _gst_428 = dg_base + goff_1_5; - __builtin_assume(__isGlobal(_gst_428)); - *(reinterpret_cast(_gst_428) + (0)) = dg_1_5; + auto _gst_460 = dg_base + goff_1_5; + __builtin_assume(__isGlobal(_gst_460)); + *(reinterpret_cast(_gst_460) + (0)) = dg_1_5; } } int t_15 = 48 + 8 * th + 6; @@ -4327,9 +4525,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_6 = offset + _tmem_load_17[6]; if (t_15 < clen) { { - auto _gst_429 = dg_base + goff_1_6; - __builtin_assume(__isGlobal(_gst_429)); - *(reinterpret_cast(_gst_429) + (0)) = dg_1_6; + auto _gst_461 = dg_base + goff_1_6; + __builtin_assume(__isGlobal(_gst_461)); + *(reinterpret_cast(_gst_461) + (0)) = dg_1_6; } } int t_16 = 48 + 8 * th + 7; @@ -4337,9 +4535,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1_7 = offset + _tmem_load_17[7]; if (t_16 < clen) { { - auto _gst_430 = dg_base + goff_1_7; - __builtin_assume(__isGlobal(_gst_430)); - *(reinterpret_cast(_gst_430) + (0)) = dg_1_7; + auto _gst_462 = dg_base + goff_1_7; + __builtin_assume(__isGlobal(_gst_462)); + *(reinterpret_cast(_gst_462) + (0)) = dg_1_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -4357,9 +4555,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_0 = offset_1250 + _tmem_load_19[0]; if (t_17 < clen) { { - auto _gst_431 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_431)); - *(reinterpret_cast(_gst_431) + (0)) = dg_0; + auto _gst_463 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_463)); + *(reinterpret_cast(_gst_463) + (0)) = dg_0; } } int t_18 = 32 + 8 * th + 1; @@ -4367,9 +4565,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_1 = offset_1250 + _tmem_load_19[1]; if (t_18 < clen) { { - auto _gst_432 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_432)); - *(reinterpret_cast(_gst_432) + (0)) = dg_1; + auto _gst_464 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_464)); + *(reinterpret_cast(_gst_464) + (0)) = dg_1; } } int t_19 = 32 + 8 * th + 2; @@ -4377,9 +4575,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_2 = offset_1250 + _tmem_load_19[2]; if (t_19 < clen) { { - auto _gst_433 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_433)); - *(reinterpret_cast(_gst_433) + (0)) = dg_2; + auto _gst_465 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_465)); + *(reinterpret_cast(_gst_465) + (0)) = dg_2; } } int t_20 = 32 + 8 * th + 3; @@ -4387,9 +4585,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_3 = offset_1250 + _tmem_load_19[3]; if (t_20 < clen) { { - auto _gst_434 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_434)); - *(reinterpret_cast(_gst_434) + (0)) = dg_3; + auto _gst_466 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_466)); + *(reinterpret_cast(_gst_466) + (0)) = dg_3; } } int t_21 = 32 + 8 * th + 4; @@ -4397,9 +4595,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_4 = offset_1250 + _tmem_load_19[4]; if (t_21 < clen) { { - auto _gst_435 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_435)); - *(reinterpret_cast(_gst_435) + (0)) = dg_4; + auto _gst_467 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_467)); + *(reinterpret_cast(_gst_467) + (0)) = dg_4; } } int t_22 = 32 + 8 * th + 5; @@ -4407,9 +4605,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_5 = offset_1250 + _tmem_load_19[5]; if (t_22 < clen) { { - auto _gst_436 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_436)); - *(reinterpret_cast(_gst_436) + (0)) = dg_5; + auto _gst_468 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_468)); + *(reinterpret_cast(_gst_468) + (0)) = dg_5; } } int t_23 = 32 + 8 * th + 6; @@ -4417,9 +4615,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_6 = offset_1250 + _tmem_load_19[6]; if (t_23 < clen) { { - auto _gst_437 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_437)); - *(reinterpret_cast(_gst_437) + (0)) = dg_6; + auto _gst_469 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_469)); + *(reinterpret_cast(_gst_469) + (0)) = dg_6; } } int t_24 = 32 + 8 * th + 7; @@ -4427,15 +4625,15 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_7 = offset_1250 + _tmem_load_19[7]; if (t_24 < clen) { { - auto _gst_438 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_438)); - *(reinterpret_cast(_gst_438) + (0)) = dg_7; + auto _gst_470 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_470)); + *(reinterpret_cast(_gst_470) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi_1248 + t_lo_1249; { - uint32_t _addr_439 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_439), "f"(off_acc[0]) : "memory"); + uint32_t _addr_471 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_471), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4495,7 +4693,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[7]; - float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, (dk2b_9 - _tmem_load_24[7]) * k_t_9) + dgf_9; + float qe_15 = q_t_1_16 * e_dg_9; + float kp_27 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_9; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_9) * _tmem_load_23[7]); + float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_24[7], k_t_9, cr_16)) + dgf_9; s_run_1302[0] = s_run_1302[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4527,7 +4730,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[6]; - float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, (dk2b_10 - _tmem_load_24[6]) * k_t_10) + dgf_10; + float qe_16 = q_t_1_17 * e_dg_10; + float kp_28 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_10; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_10) * _tmem_load_23[6]); + float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_24[6], k_t_10, cr_17)) + dgf_10; s_run_1302[0] = s_run_1302[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4559,7 +4767,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[5]; - float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, (dk2b_11 - _tmem_load_24[5]) * k_t_11) + dgf_11; + float qe_17 = q_t_1_18 * e_dg_11; + float kp_29 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_11; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_11) * _tmem_load_23[5]); + float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_24[5], k_t_11, cr_18)) + dgf_11; s_run_1302[0] = s_run_1302[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4591,7 +4804,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[4]; - float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, (dk2b_12 - _tmem_load_24[4]) * k_t_12) + dgf_12; + float qe_18 = q_t_1_19 * e_dg_12; + float kp_30 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_12; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_12) * _tmem_load_23[4]); + float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_24[4], k_t_12, cr_19)) + dgf_12; s_run_1302[0] = s_run_1302[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4623,7 +4841,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[3]; - float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, (dk2b_13 - _tmem_load_24[3]) * k_t_13) + dgf_13; + float qe_19 = q_t_1_20 * e_dg_13; + float kp_31 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_13; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_13) * _tmem_load_23[3]); + float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_24[3], k_t_13, cr_20)) + dgf_13; s_run_1302[0] = s_run_1302[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4655,7 +4878,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[2]; - float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, (dk2b_14 - _tmem_load_24[2]) * k_t_14) + dgf_14; + float qe_20 = q_t_1_21 * e_dg_14; + float kp_32 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_14; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_14) * _tmem_load_23[2]); + float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_24[2], k_t_14, cr_21)) + dgf_14; s_run_1302[0] = s_run_1302[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4687,7 +4915,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[1]; - float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, (dk2b_15 - _tmem_load_24[1]) * k_t_15) + dgf_15; + float qe_21 = q_t_1_22 * e_dg_15; + float kp_33 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_15; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_15) * _tmem_load_23[1]); + float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_24[1], k_t_15, cr_22)) + dgf_15; s_run_1302[0] = s_run_1302[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4719,7 +4952,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_24[0]; - float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, (dk2b_16 - _tmem_load_24[0]) * k_t_16) + dgf_16; + float qe_22 = q_t_1_23 * e_dg_16; + float kp_34 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_16; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_16) * _tmem_load_23[0]); + float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, __fmaf_rn(dk2b_16 - _tmem_load_24[0], k_t_16, cr_23)) + dgf_16; s_run_1302[0] = s_run_1302[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4734,40 +4972,40 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; float send_7 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_440 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_440; + float _shfl_xor_sync_472 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_472; float send_8 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_441 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_441; + float _shfl_xor_sync_473 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_473; float send_9 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_442 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_442; + float _shfl_xor_sync_474 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_474; float send_10 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_443 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_443; + float _shfl_xor_sync_475 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_475; float v2_1480[2]; float send_11 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_444 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_444; + float _shfl_xor_sync_476 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_476; float send_12 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_445 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_445; + float _shfl_xor_sync_477 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_477; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_446 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_446; - float _shfl_xor_sync_447 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_447; - float _shfl_xor_sync_448 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_448; + float _shfl_xor_sync_478 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_478; + float _shfl_xor_sync_479 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_479; + float _shfl_xor_sync_480 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_480; asm volatile("tcgen05.fence::before_thread_sync;"); int ew_2 = 81920 - 32768 * th >> 2; float nz_1 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); bool live0_1 = 16 + 8 * th + qd < clen; bool live1_1 = 16 + 8 * th + qd + 4 < clen; - int _min_449 = ((16 + 8 * th + qd) < (clen - 1) ? (16 + 8 * th + qd) : (clen - 1)); - long long rowa_1 = ext0 + (long long)_min_449; - int _min_450 = ((16 + 8 * th + qd + 4) < (clen - 1) ? (16 + 8 * th + qd + 4) : (clen - 1)); - long long rowb_1 = ext0 + (long long)_min_450; + int _min_481 = ((16 + 8 * th + qd) < (clen - 1) ? (16 + 8 * th + qd) : (clen - 1)); + long long rowa_1 = ext0 + (long long)_min_481; + int _min_482 = ((16 + 8 * th + qd + 4) < (clen - 1) ? (16 + 8 * th + qd + 4) : (clen - 1)); + long long rowb_1 = ext0 + (long long)_min_482; float xqa_1[4]; float xka_1[4]; float xqb_1[4]; @@ -4795,52 +5033,52 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM : "r"(pool_f32_addr + (unsigned int)(4 * (ew_2 + 1024 + 128 * (qd + 4) + 4 * lane_1)))); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); { - uint2 _vld_451; - _vld_451 = *reinterpret_cast(q_raw + (rowa_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_451 = reinterpret_cast(&_vld_451); + uint2 _vld_483; + _vld_483 = *reinterpret_cast(q_raw + (rowa_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_483 = reinterpret_cast(&_vld_483); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqa_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_451[_pair]) << 16); - (&xqa_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_451[_pair]) & 0xffff0000u); + (&xqa_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_483[_pair]) << 16); + (&xqa_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_483[_pair]) & 0xffff0000u); } } { rqa_1[0] = *reinterpret_cast(qk_rstd + ((0 + rowa_1) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_452; - _vld_452 = *reinterpret_cast(k_raw + (rowa_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_452 = reinterpret_cast(&_vld_452); + uint2 _vld_484; + _vld_484 = *reinterpret_cast(k_raw + (rowa_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_484 = reinterpret_cast(&_vld_484); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xka_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_452[_pair]) << 16); - (&xka_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_452[_pair]) & 0xffff0000u); + (&xka_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_484[_pair]) << 16); + (&xka_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_484[_pair]) & 0xffff0000u); } } { rka_1[0] = *reinterpret_cast(qk_rstd + (((long long)num_tokens + rowa_1) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_453; - _vld_453 = *reinterpret_cast(q_raw + (rowb_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_453 = reinterpret_cast(&_vld_453); + uint2 _vld_485; + _vld_485 = *reinterpret_cast(q_raw + (rowb_1 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_485 = reinterpret_cast(&_vld_485); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqb_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_453[_pair]) << 16); - (&xqb_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_453[_pair]) & 0xffff0000u); + (&xqb_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_485[_pair]) << 16); + (&xqb_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_485[_pair]) & 0xffff0000u); } } { rqb_1[0] = *reinterpret_cast(qk_rstd + ((0 + rowb_1) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_454; - _vld_454 = *reinterpret_cast(k_raw + (rowb_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_454 = reinterpret_cast(&_vld_454); + uint2 _vld_486; + _vld_486 = *reinterpret_cast(k_raw + (rowb_1 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_486 = reinterpret_cast(&_vld_486); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xkb_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_454[_pair]) << 16); - (&xkb_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_454[_pair]) & 0xffff0000u); + (&xkb_1[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_486[_pair]) << 16); + (&xkb_1[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_486[_pair]) & 0xffff0000u); } } { @@ -4880,106 +5118,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_2[4]; float p_2[4]; y_2[0] = mul_f32_rn_noftz(xq_2[0], rq_2[0]); - float _fma_455; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_455) : "f"(gq_2[0]), "f"(y_2[0]), "f"(nz_1)); - p_2[0] = _fma_455; + float _fma_487; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_487) : "f"(gq_2[0]), "f"(y_2[0]), "f"(nz_1)); + p_2[0] = _fma_487; y_2[1] = mul_f32_rn_noftz(xq_2[1], rq_2[0]); - float _fma_456; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_456) : "f"(gq_2[1]), "f"(y_2[1]), "f"(nz_1)); - p_2[1] = _fma_456; + float _fma_488; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_488) : "f"(gq_2[1]), "f"(y_2[1]), "f"(nz_1)); + p_2[1] = _fma_488; y_2[2] = mul_f32_rn_noftz(xq_2[2], rq_2[0]); - float _fma_457; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_457) : "f"(gq_2[2]), "f"(y_2[2]), "f"(nz_1)); - p_2[2] = _fma_457; + float _fma_489; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_489) : "f"(gq_2[2]), "f"(y_2[2]), "f"(nz_1)); + p_2[2] = _fma_489; y_2[3] = mul_f32_rn_noftz(xq_2[3], rq_2[0]); - float _fma_458; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_458) : "f"(gq_2[3]), "f"(y_2[3]), "f"(nz_1)); - p_2[3] = _fma_458; - float _shfl_xor_sync_460 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 4); - float _add_459; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_459) : "f"(p_2[0]), "f"(_shfl_xor_sync_460)); - p_2[0] = _add_459; - float _shfl_xor_sync_462 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 4); - float _add_461; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_461) : "f"(p_2[1]), "f"(_shfl_xor_sync_462)); - p_2[1] = _add_461; - float _shfl_xor_sync_464 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 4); - float _add_463; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_463) : "f"(p_2[2]), "f"(_shfl_xor_sync_464)); - p_2[2] = _add_463; - float _shfl_xor_sync_466 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 4); - float _add_465; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_465) : "f"(p_2[3]), "f"(_shfl_xor_sync_466)); - p_2[3] = _add_465; - float _shfl_xor_sync_468 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 2); - float _add_467; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_467) : "f"(p_2[0]), "f"(_shfl_xor_sync_468)); - p_2[0] = _add_467; - float _shfl_xor_sync_470 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 2); - float _add_469; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_469) : "f"(p_2[1]), "f"(_shfl_xor_sync_470)); - p_2[1] = _add_469; - float _shfl_xor_sync_472 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 2); - float _add_471; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_471) : "f"(p_2[2]), "f"(_shfl_xor_sync_472)); - p_2[2] = _add_471; - float _shfl_xor_sync_474 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 2); - float _add_473; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_473) : "f"(p_2[3]), "f"(_shfl_xor_sync_474)); - p_2[3] = _add_473; - float _shfl_xor_sync_476 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 1); - float _add_475; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_475) : "f"(p_2[0]), "f"(_shfl_xor_sync_476)); - p_2[0] = _add_475; - float _shfl_xor_sync_478 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 1); - float _add_477; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_477) : "f"(p_2[1]), "f"(_shfl_xor_sync_478)); - p_2[1] = _add_477; - float _shfl_xor_sync_480 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 1); - float _add_479; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_479) : "f"(p_2[2]), "f"(_shfl_xor_sync_480)); - p_2[2] = _add_479; - float _shfl_xor_sync_482 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 1); - float _add_481; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_481) : "f"(p_2[3]), "f"(_shfl_xor_sync_482)); - p_2[3] = _add_481; - float _add_483; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_483) : "f"(p_2[0]), "f"(p_2[2])); - float _add_484; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_484) : "f"(p_2[1]), "f"(p_2[3])); + float _fma_490; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_490) : "f"(gq_2[3]), "f"(y_2[3]), "f"(nz_1)); + p_2[3] = _fma_490; + float _shfl_xor_sync_492 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 4); + float _add_491; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_491) : "f"(p_2[0]), "f"(_shfl_xor_sync_492)); + p_2[0] = _add_491; + float _shfl_xor_sync_494 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 4); + float _add_493; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_493) : "f"(p_2[1]), "f"(_shfl_xor_sync_494)); + p_2[1] = _add_493; + float _shfl_xor_sync_496 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 4); + float _add_495; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_495) : "f"(p_2[2]), "f"(_shfl_xor_sync_496)); + p_2[2] = _add_495; + float _shfl_xor_sync_498 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 4); + float _add_497; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_497) : "f"(p_2[3]), "f"(_shfl_xor_sync_498)); + p_2[3] = _add_497; + float _shfl_xor_sync_500 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 2); + float _add_499; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_499) : "f"(p_2[0]), "f"(_shfl_xor_sync_500)); + p_2[0] = _add_499; + float _shfl_xor_sync_502 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 2); + float _add_501; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_501) : "f"(p_2[1]), "f"(_shfl_xor_sync_502)); + p_2[1] = _add_501; + float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 2); + float _add_503; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_503) : "f"(p_2[2]), "f"(_shfl_xor_sync_504)); + p_2[2] = _add_503; + float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 2); + float _add_505; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_505) : "f"(p_2[3]), "f"(_shfl_xor_sync_506)); + p_2[3] = _add_505; + float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, p_2[0], 1); + float _add_507; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_507) : "f"(p_2[0]), "f"(_shfl_xor_sync_508)); + p_2[0] = _add_507; + float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, p_2[1], 1); + float _add_509; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_509) : "f"(p_2[1]), "f"(_shfl_xor_sync_510)); + p_2[1] = _add_509; + float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, p_2[2], 1); + float _add_511; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_511) : "f"(p_2[2]), "f"(_shfl_xor_sync_512)); + p_2[2] = _add_511; + float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, p_2[3], 1); + float _add_513; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_513) : "f"(p_2[3]), "f"(_shfl_xor_sync_514)); + p_2[3] = _add_513; + float _add_515; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_515) : "f"(p_2[0]), "f"(p_2[2])); + float _add_516; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_516) : "f"(p_2[1]), "f"(p_2[3])); float sq_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2) : "f"(_add_483), "f"(_add_484)); - float _shfl_sync_487 = __shfl_sync(0xFFFFFFFF, sq_2, 0); - float _shfl_sync_488 = __shfl_sync(0xFFFFFFFF, sq_2, 8); - float _add_486; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_486) : "f"(_shfl_sync_487), "f"(_shfl_sync_488)); - float _shfl_sync_489 = __shfl_sync(0xFFFFFFFF, sq_2, 16); - float _add_485; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_485) : "f"(_add_486), "f"(_shfl_sync_489)); - float _shfl_sync_490 = __shfl_sync(0xFFFFFFFF, sq_2, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2) : "f"(_add_515), "f"(_add_516)); + float _shfl_sync_519 = __shfl_sync(0xFFFFFFFF, sq_2, 0); + float _shfl_sync_520 = __shfl_sync(0xFFFFFFFF, sq_2, 8); + float _add_518; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_518) : "f"(_shfl_sync_519), "f"(_shfl_sync_520)); + float _shfl_sync_521 = __shfl_sync(0xFFFFFFFF, sq_2, 16); + float _add_517; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_517) : "f"(_add_518), "f"(_shfl_sync_521)); + float _shfl_sync_522 = __shfl_sync(0xFFFFFFFF, sq_2, 24); float dot_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_485), "f"(_shfl_sync_490)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2) : "f"(_add_517), "f"(_shfl_sync_522)); float o_2[4]; - float _fma_492; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_492) : "f"(y_2[0]), "f"(dot_2), "f"(nz_1)); - float _sub_491; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_491) : "f"(gq_2[0]), "f"(_fma_492)); - o_2[0] = mul_f32_rn_noftz(rq_2[0], _sub_491); - float _fma_494; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_494) : "f"(y_2[1]), "f"(dot_2), "f"(nz_1)); - float _sub_493; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_493) : "f"(gq_2[1]), "f"(_fma_494)); - o_2[1] = mul_f32_rn_noftz(rq_2[0], _sub_493); - float _fma_496; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_496) : "f"(y_2[2]), "f"(dot_2), "f"(nz_1)); - float _sub_495; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_495) : "f"(gq_2[2]), "f"(_fma_496)); - o_2[2] = mul_f32_rn_noftz(rq_2[0], _sub_495); - float _fma_498; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_498) : "f"(y_2[3]), "f"(dot_2), "f"(nz_1)); - float _sub_497; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_497) : "f"(gq_2[3]), "f"(_fma_498)); - o_2[3] = mul_f32_rn_noftz(rq_2[0], _sub_497); + float _fma_524; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_524) : "f"(y_2[0]), "f"(dot_2), "f"(nz_1)); + float _sub_523; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_523) : "f"(gq_2[0]), "f"(_fma_524)); + o_2[0] = mul_f32_rn_noftz(rq_2[0], _sub_523); + float _fma_526; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_526) : "f"(y_2[1]), "f"(dot_2), "f"(nz_1)); + float _sub_525; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_525) : "f"(gq_2[1]), "f"(_fma_526)); + o_2[1] = mul_f32_rn_noftz(rq_2[0], _sub_525); + float _fma_528; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_528) : "f"(y_2[2]), "f"(dot_2), "f"(nz_1)); + float _sub_527; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_527) : "f"(gq_2[2]), "f"(_fma_528)); + o_2[2] = mul_f32_rn_noftz(rq_2[0], _sub_527); + float _fma_530; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_530) : "f"(y_2[3]), "f"(dot_2), "f"(nz_1)); + float _sub_529; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_529) : "f"(gq_2[3]), "f"(_fma_530)); + o_2[3] = mul_f32_rn_noftz(rq_2[0], _sub_529); unsigned int ob_2[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -4990,106 +5228,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_2_1[4]; float p_2_1[4]; y_2_1[0] = mul_f32_rn_noftz(xk_2[0], rk_2[0]); - float _fma_499; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_499) : "f"(gk_2[0]), "f"(y_2_1[0]), "f"(nz_1)); - p_2_1[0] = _fma_499; + float _fma_531; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_531) : "f"(gk_2[0]), "f"(y_2_1[0]), "f"(nz_1)); + p_2_1[0] = _fma_531; y_2_1[1] = mul_f32_rn_noftz(xk_2[1], rk_2[0]); - float _fma_500; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_500) : "f"(gk_2[1]), "f"(y_2_1[1]), "f"(nz_1)); - p_2_1[1] = _fma_500; + float _fma_532; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_532) : "f"(gk_2[1]), "f"(y_2_1[1]), "f"(nz_1)); + p_2_1[1] = _fma_532; y_2_1[2] = mul_f32_rn_noftz(xk_2[2], rk_2[0]); - float _fma_501; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_501) : "f"(gk_2[2]), "f"(y_2_1[2]), "f"(nz_1)); - p_2_1[2] = _fma_501; + float _fma_533; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_533) : "f"(gk_2[2]), "f"(y_2_1[2]), "f"(nz_1)); + p_2_1[2] = _fma_533; y_2_1[3] = mul_f32_rn_noftz(xk_2[3], rk_2[0]); - float _fma_502; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_502) : "f"(gk_2[3]), "f"(y_2_1[3]), "f"(nz_1)); - p_2_1[3] = _fma_502; - float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 4); - float _add_503; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_503) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_504)); - p_2_1[0] = _add_503; - float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 4); - float _add_505; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_505) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_506)); - p_2_1[1] = _add_505; - float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 4); - float _add_507; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_507) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_508)); - p_2_1[2] = _add_507; - float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 4); - float _add_509; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_509) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_510)); - p_2_1[3] = _add_509; - float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 2); - float _add_511; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_511) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_512)); - p_2_1[0] = _add_511; - float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 2); - float _add_513; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_513) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_514)); - p_2_1[1] = _add_513; - float _shfl_xor_sync_516 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 2); - float _add_515; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_515) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_516)); - p_2_1[2] = _add_515; - float _shfl_xor_sync_518 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 2); - float _add_517; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_517) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_518)); - p_2_1[3] = _add_517; - float _shfl_xor_sync_520 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 1); - float _add_519; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_519) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_520)); - p_2_1[0] = _add_519; - float _shfl_xor_sync_522 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 1); - float _add_521; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_521) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_522)); - p_2_1[1] = _add_521; - float _shfl_xor_sync_524 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 1); - float _add_523; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_523) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_524)); - p_2_1[2] = _add_523; - float _shfl_xor_sync_526 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 1); - float _add_525; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_525) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_526)); - p_2_1[3] = _add_525; - float _add_527; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_527) : "f"(p_2_1[0]), "f"(p_2_1[2])); - float _add_528; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_528) : "f"(p_2_1[1]), "f"(p_2_1[3])); + float _fma_534; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_534) : "f"(gk_2[3]), "f"(y_2_1[3]), "f"(nz_1)); + p_2_1[3] = _fma_534; + float _shfl_xor_sync_536 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 4); + float _add_535; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_535) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_536)); + p_2_1[0] = _add_535; + float _shfl_xor_sync_538 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 4); + float _add_537; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_537) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_538)); + p_2_1[1] = _add_537; + float _shfl_xor_sync_540 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 4); + float _add_539; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_539) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_540)); + p_2_1[2] = _add_539; + float _shfl_xor_sync_542 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 4); + float _add_541; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_541) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_542)); + p_2_1[3] = _add_541; + float _shfl_xor_sync_544 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 2); + float _add_543; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_543) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_544)); + p_2_1[0] = _add_543; + float _shfl_xor_sync_546 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 2); + float _add_545; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_545) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_546)); + p_2_1[1] = _add_545; + float _shfl_xor_sync_548 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 2); + float _add_547; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_547) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_548)); + p_2_1[2] = _add_547; + float _shfl_xor_sync_550 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 2); + float _add_549; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_549) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_550)); + p_2_1[3] = _add_549; + float _shfl_xor_sync_552 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[0], 1); + float _add_551; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_551) : "f"(p_2_1[0]), "f"(_shfl_xor_sync_552)); + p_2_1[0] = _add_551; + float _shfl_xor_sync_554 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[1], 1); + float _add_553; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_553) : "f"(p_2_1[1]), "f"(_shfl_xor_sync_554)); + p_2_1[1] = _add_553; + float _shfl_xor_sync_556 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[2], 1); + float _add_555; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_555) : "f"(p_2_1[2]), "f"(_shfl_xor_sync_556)); + p_2_1[2] = _add_555; + float _shfl_xor_sync_558 = __shfl_xor_sync(0xFFFFFFFF, p_2_1[3], 1); + float _add_557; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_557) : "f"(p_2_1[3]), "f"(_shfl_xor_sync_558)); + p_2_1[3] = _add_557; + float _add_559; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_559) : "f"(p_2_1[0]), "f"(p_2_1[2])); + float _add_560; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_560) : "f"(p_2_1[1]), "f"(p_2_1[3])); float sq_2_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2_1) : "f"(_add_527), "f"(_add_528)); - float _shfl_sync_531 = __shfl_sync(0xFFFFFFFF, sq_2_1, 0); - float _shfl_sync_532 = __shfl_sync(0xFFFFFFFF, sq_2_1, 8); - float _add_530; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_530) : "f"(_shfl_sync_531), "f"(_shfl_sync_532)); - float _shfl_sync_533 = __shfl_sync(0xFFFFFFFF, sq_2_1, 16); - float _add_529; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_529) : "f"(_add_530), "f"(_shfl_sync_533)); - float _shfl_sync_534 = __shfl_sync(0xFFFFFFFF, sq_2_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_2_1) : "f"(_add_559), "f"(_add_560)); + float _shfl_sync_563 = __shfl_sync(0xFFFFFFFF, sq_2_1, 0); + float _shfl_sync_564 = __shfl_sync(0xFFFFFFFF, sq_2_1, 8); + float _add_562; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_562) : "f"(_shfl_sync_563), "f"(_shfl_sync_564)); + float _shfl_sync_565 = __shfl_sync(0xFFFFFFFF, sq_2_1, 16); + float _add_561; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_561) : "f"(_add_562), "f"(_shfl_sync_565)); + float _shfl_sync_566 = __shfl_sync(0xFFFFFFFF, sq_2_1, 24); float dot_2_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2_1) : "f"(_add_529), "f"(_shfl_sync_534)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_2_1) : "f"(_add_561), "f"(_shfl_sync_566)); float o_2_1[4]; - float _fma_536; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_536) : "f"(y_2_1[0]), "f"(dot_2_1), "f"(nz_1)); - float _sub_535; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_535) : "f"(gk_2[0]), "f"(_fma_536)); - o_2_1[0] = mul_f32_rn_noftz(rk_2[0], _sub_535); - float _fma_538; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_538) : "f"(y_2_1[1]), "f"(dot_2_1), "f"(nz_1)); - float _sub_537; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_537) : "f"(gk_2[1]), "f"(_fma_538)); - o_2_1[1] = mul_f32_rn_noftz(rk_2[0], _sub_537); - float _fma_540; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_540) : "f"(y_2_1[2]), "f"(dot_2_1), "f"(nz_1)); - float _sub_539; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_539) : "f"(gk_2[2]), "f"(_fma_540)); - o_2_1[2] = mul_f32_rn_noftz(rk_2[0], _sub_539); - float _fma_542; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_542) : "f"(y_2_1[3]), "f"(dot_2_1), "f"(nz_1)); - float _sub_541; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_541) : "f"(gk_2[3]), "f"(_fma_542)); - o_2_1[3] = mul_f32_rn_noftz(rk_2[0], _sub_541); + float _fma_568; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_568) : "f"(y_2_1[0]), "f"(dot_2_1), "f"(nz_1)); + float _sub_567; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_567) : "f"(gk_2[0]), "f"(_fma_568)); + o_2_1[0] = mul_f32_rn_noftz(rk_2[0], _sub_567); + float _fma_570; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_570) : "f"(y_2_1[1]), "f"(dot_2_1), "f"(nz_1)); + float _sub_569; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_569) : "f"(gk_2[1]), "f"(_fma_570)); + o_2_1[1] = mul_f32_rn_noftz(rk_2[0], _sub_569); + float _fma_572; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_572) : "f"(y_2_1[2]), "f"(dot_2_1), "f"(nz_1)); + float _sub_571; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(gk_2[2]), "f"(_fma_572)); + o_2_1[2] = mul_f32_rn_noftz(rk_2[0], _sub_571); + float _fma_574; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_574) : "f"(y_2_1[3]), "f"(dot_2_1), "f"(nz_1)); + float _sub_573; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_573) : "f"(gk_2[3]), "f"(_fma_574)); + o_2_1[3] = mul_f32_rn_noftz(rk_2[0], _sub_573); unsigned int ob_2_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -5100,8 +5338,8 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM } asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_543 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_543), "f"(tot_1471) : "memory"); + uint32_t _addr_575 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_575), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5130,9 +5368,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_8 = offset_1488 + _tmem_load_25[0]; if (t_33 < clen) { { - auto _gst_544 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_544)); - *(reinterpret_cast(_gst_544) + (0)) = dg_8; + auto _gst_576 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_576)); + *(reinterpret_cast(_gst_576) + (0)) = dg_8; } } int t_34 = 16 + 8 * th + 1; @@ -5140,9 +5378,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_9 = offset_1488 + _tmem_load_25[1]; if (t_34 < clen) { { - auto _gst_545 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_545)); - *(reinterpret_cast(_gst_545) + (0)) = dg_9; + auto _gst_577 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_577)); + *(reinterpret_cast(_gst_577) + (0)) = dg_9; } } int t_35 = 16 + 8 * th + 2; @@ -5150,9 +5388,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_10 = offset_1488 + _tmem_load_25[2]; if (t_35 < clen) { { - auto _gst_546 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_546)); - *(reinterpret_cast(_gst_546) + (0)) = dg_10; + auto _gst_578 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_578)); + *(reinterpret_cast(_gst_578) + (0)) = dg_10; } } int t_36 = 16 + 8 * th + 3; @@ -5160,9 +5398,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_11 = offset_1488 + _tmem_load_25[3]; if (t_36 < clen) { { - auto _gst_547 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_547)); - *(reinterpret_cast(_gst_547) + (0)) = dg_11; + auto _gst_579 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_579)); + *(reinterpret_cast(_gst_579) + (0)) = dg_11; } } int t_37 = 16 + 8 * th + 4; @@ -5170,9 +5408,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_12 = offset_1488 + _tmem_load_25[4]; if (t_37 < clen) { { - auto _gst_548 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_548)); - *(reinterpret_cast(_gst_548) + (0)) = dg_12; + auto _gst_580 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_580)); + *(reinterpret_cast(_gst_580) + (0)) = dg_12; } } int t_38 = 16 + 8 * th + 5; @@ -5180,9 +5418,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_13 = offset_1488 + _tmem_load_25[5]; if (t_38 < clen) { { - auto _gst_549 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_549)); - *(reinterpret_cast(_gst_549) + (0)) = dg_13; + auto _gst_581 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_581)); + *(reinterpret_cast(_gst_581) + (0)) = dg_13; } } int t_39 = 16 + 8 * th + 6; @@ -5190,9 +5428,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_14 = offset_1488 + _tmem_load_25[6]; if (t_39 < clen) { { - auto _gst_550 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_550)); - *(reinterpret_cast(_gst_550) + (0)) = dg_14; + auto _gst_582 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_582)); + *(reinterpret_cast(_gst_582) + (0)) = dg_14; } } int t_40 = 16 + 8 * th + 7; @@ -5200,15 +5438,15 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_15 = offset_1488 + _tmem_load_25[7]; if (t_40 < clen) { { - auto _gst_551 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_551)); - *(reinterpret_cast(_gst_551) + (0)) = dg_15; + auto _gst_583 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_583)); + *(reinterpret_cast(_gst_583) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_1486 + t_lo_1487; { - uint32_t _addr_552 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_552), "f"(off_acc[0]) : "memory"); + uint32_t _addr_584 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_584), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -5262,7 +5500,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[7]; - float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, (dk2b_17 - _tmem_load_28[7]) * k_t_17) + dgf_17; + float qe_23 = q_t_1_24 * e_dg_17; + float kp_35 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_35; + float kbe_23 = (float)kb_23 * e_dg_17; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_35 * e_dg_17) * _tmem_load_27[7]); + float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, __fmaf_rn(dk2b_17 - _tmem_load_28[7], k_t_17, cr_24)) + dgf_17; s_run_1524[0] = s_run_1524[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5294,7 +5537,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[6]; - float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, (dk2b_18 - _tmem_load_28[6]) * k_t_18) + dgf_18; + float qe_24 = q_t_1_25 * e_dg_18; + float kp_36 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_36; + float kbe_24 = (float)kb_24 * e_dg_18; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_36 * e_dg_18) * _tmem_load_27[6]); + float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, __fmaf_rn(dk2b_18 - _tmem_load_28[6], k_t_18, cr_25)) + dgf_18; s_run_1524[0] = s_run_1524[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5326,7 +5574,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[5]; - float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, (dk2b_19 - _tmem_load_28[5]) * k_t_19) + dgf_19; + float qe_25 = q_t_1_26 * e_dg_19; + float kp_37 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_37; + float kbe_25 = (float)kb_25 * e_dg_19; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_37 * e_dg_19) * _tmem_load_27[5]); + float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, __fmaf_rn(dk2b_19 - _tmem_load_28[5], k_t_19, cr_26)) + dgf_19; s_run_1524[0] = s_run_1524[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5358,7 +5611,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[4]; - float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, (dk2b_20 - _tmem_load_28[4]) * k_t_20) + dgf_20; + float qe_26 = q_t_1_27 * e_dg_20; + float kp_38 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_38; + float kbe_26 = (float)kb_26 * e_dg_20; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_38 * e_dg_20) * _tmem_load_27[4]); + float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, __fmaf_rn(dk2b_20 - _tmem_load_28[4], k_t_20, cr_27)) + dgf_20; s_run_1524[0] = s_run_1524[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5390,7 +5648,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[3]; - float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, (dk2b_21 - _tmem_load_28[3]) * k_t_21) + dgf_21; + float qe_27 = q_t_1_28 * e_dg_21; + float kp_39 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_39; + float kbe_27 = (float)kb_27 * e_dg_21; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_39 * e_dg_21) * _tmem_load_27[3]); + float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, __fmaf_rn(dk2b_21 - _tmem_load_28[3], k_t_21, cr_28)) + dgf_21; s_run_1524[0] = s_run_1524[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5422,7 +5685,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[2]; - float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, (dk2b_22 - _tmem_load_28[2]) * k_t_22) + dgf_22; + float qe_28 = q_t_1_29 * e_dg_22; + float kp_40 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_40; + float kbe_28 = (float)kb_28 * e_dg_22; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_40 * e_dg_22) * _tmem_load_27[2]); + float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, __fmaf_rn(dk2b_22 - _tmem_load_28[2], k_t_22, cr_29)) + dgf_22; s_run_1524[0] = s_run_1524[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5454,7 +5722,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[1]; - float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, (dk2b_23 - _tmem_load_28[1]) * k_t_23) + dgf_23; + float qe_29 = q_t_1_30 * e_dg_23; + float kp_41 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_41; + float kbe_29 = (float)kb_29 * e_dg_23; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_41 * e_dg_23) * _tmem_load_27[1]); + float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, __fmaf_rn(dk2b_23 - _tmem_load_28[1], k_t_23, cr_30)) + dgf_23; s_run_1524[0] = s_run_1524[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5486,7 +5759,12 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + _tmem_load_28[0]; - float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, (dk2b_24 - _tmem_load_28[0]) * k_t_24) + dgf_24; + float qe_30 = q_t_1_31 * e_dg_24; + float kp_42 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_42; + float kbe_30 = (float)kb_30 * e_dg_24; + float cr_31 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_42 * e_dg_24) * _tmem_load_27[0]); + float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, __fmaf_rn(dk2b_24 - _tmem_load_28[0], k_t_24, cr_31)) + dgf_24; s_run_1524[0] = s_run_1524[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -5501,39 +5779,39 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; float send_13 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_553 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_553; + float _shfl_xor_sync_585 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_585; float send_14 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_554 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_554; + float _shfl_xor_sync_586 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_586; float send_15 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_555 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_555; + float _shfl_xor_sync_587 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_587; float send_16 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_556 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_556; + float _shfl_xor_sync_588 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_588; float v2_1702[2]; float send_17 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_557 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_557; + float _shfl_xor_sync_589 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_589; float send_18 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_558 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_558; + float _shfl_xor_sync_590 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_590; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_559 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_559; - float _shfl_xor_sync_560 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_560; - float _shfl_xor_sync_561 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_561; + float _shfl_xor_sync_591 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_591; + float _shfl_xor_sync_592 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_592; + float _shfl_xor_sync_593 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_593; int ew_3 = 98304 - 32768 * th >> 2; float nz_2 = __uint_as_float((unsigned int)num_heads >> 31 | 2147483648u); bool live0_2 = 8 * th + qd < clen; bool live1_2 = 8 * th + qd + 4 < clen; - int _min_562 = ((8 * th + qd) < (clen - 1) ? (8 * th + qd) : (clen - 1)); - long long rowa_2 = ext0 + (long long)_min_562; - int _min_563 = ((8 * th + qd + 4) < (clen - 1) ? (8 * th + qd + 4) : (clen - 1)); - long long rowb_2 = ext0 + (long long)_min_563; + int _min_594 = ((8 * th + qd) < (clen - 1) ? (8 * th + qd) : (clen - 1)); + long long rowa_2 = ext0 + (long long)_min_594; + int _min_595 = ((8 * th + qd + 4) < (clen - 1) ? (8 * th + qd + 4) : (clen - 1)); + long long rowb_2 = ext0 + (long long)_min_595; float xqa_2[4]; float xka_2[4]; float xqb_2[4]; @@ -5560,52 +5838,52 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM : "=r"(*reinterpret_cast(&gkb_2[0])), "=r"(*reinterpret_cast(&gkb_2[(0) + 1])), "=r"(*reinterpret_cast(&gkb_2[(0) + 2])), "=r"(*reinterpret_cast(&gkb_2[(0) + 3])) : "r"(pool_f32_addr + (unsigned int)(4 * (ew_3 + 1024 + 128 * (qd + 4) + 4 * lane_1)))); { - uint2 _vld_564; - _vld_564 = *reinterpret_cast(q_raw + (rowa_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_564 = reinterpret_cast(&_vld_564); + uint2 _vld_596; + _vld_596 = *reinterpret_cast(q_raw + (rowa_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_596 = reinterpret_cast(&_vld_596); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqa_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_564[_pair]) << 16); - (&xqa_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_564[_pair]) & 0xffff0000u); + (&xqa_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_596[_pair]) << 16); + (&xqa_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_596[_pair]) & 0xffff0000u); } } { rqa_2[0] = *reinterpret_cast(qk_rstd + ((0 + rowa_2) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_565; - _vld_565 = *reinterpret_cast(k_raw + (rowa_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_565 = reinterpret_cast(&_vld_565); + uint2 _vld_597; + _vld_597 = *reinterpret_cast(k_raw + (rowa_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_597 = reinterpret_cast(&_vld_597); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xka_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_565[_pair]) << 16); - (&xka_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_565[_pair]) & 0xffff0000u); + (&xka_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_597[_pair]) << 16); + (&xka_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_597[_pair]) & 0xffff0000u); } } { rka_2[0] = *reinterpret_cast(qk_rstd + (((long long)num_tokens + rowa_2) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_566; - _vld_566 = *reinterpret_cast(q_raw + (rowb_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_566 = reinterpret_cast(&_vld_566); + uint2 _vld_598; + _vld_598 = *reinterpret_cast(q_raw + (rowb_2 * (long long)q_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_598 = reinterpret_cast(&_vld_598); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xqb_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_566[_pair]) << 16); - (&xqb_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_566[_pair]) & 0xffff0000u); + (&xqb_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_598[_pair]) << 16); + (&xqb_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_598[_pair]) & 0xffff0000u); } } { rqb_2[0] = *reinterpret_cast(qk_rstd + ((0 + rowb_2) * (long long)num_qk_heads + (long long)qk_head) + 0); } { - uint2 _vld_567; - _vld_567 = *reinterpret_cast(k_raw + (rowb_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); - uint32_t* _vpairs_567 = reinterpret_cast(&_vld_567); + uint2 _vld_599; + _vld_599 = *reinterpret_cast(k_raw + (rowb_2 * (long long)k_raw_row_stride + (long long)(qk_head * 128 + 4 * lane_1)) + 0); + uint32_t* _vpairs_599 = reinterpret_cast(&_vld_599); #pragma unroll for (int _pair = 0; _pair < 2; _pair++) { - (&xkb_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_567[_pair]) << 16); - (&xkb_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_567[_pair]) & 0xffff0000u); + (&xkb_2[0 + _pair * 2])[0] = __uint_as_float(static_cast(_vpairs_599[_pair]) << 16); + (&xkb_2[0 + _pair * 2])[1] = __uint_as_float(static_cast(_vpairs_599[_pair]) & 0xffff0000u); } } { @@ -5645,106 +5923,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_3[4]; float p_3[4]; y_3[0] = mul_f32_rn_noftz(xq_3[0], rq_3[0]); - float _fma_568; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_568) : "f"(gq_3[0]), "f"(y_3[0]), "f"(nz_2)); - p_3[0] = _fma_568; + float _fma_600; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_600) : "f"(gq_3[0]), "f"(y_3[0]), "f"(nz_2)); + p_3[0] = _fma_600; y_3[1] = mul_f32_rn_noftz(xq_3[1], rq_3[0]); - float _fma_569; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_569) : "f"(gq_3[1]), "f"(y_3[1]), "f"(nz_2)); - p_3[1] = _fma_569; + float _fma_601; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_601) : "f"(gq_3[1]), "f"(y_3[1]), "f"(nz_2)); + p_3[1] = _fma_601; y_3[2] = mul_f32_rn_noftz(xq_3[2], rq_3[0]); - float _fma_570; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_570) : "f"(gq_3[2]), "f"(y_3[2]), "f"(nz_2)); - p_3[2] = _fma_570; + float _fma_602; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_602) : "f"(gq_3[2]), "f"(y_3[2]), "f"(nz_2)); + p_3[2] = _fma_602; y_3[3] = mul_f32_rn_noftz(xq_3[3], rq_3[0]); - float _fma_571; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_571) : "f"(gq_3[3]), "f"(y_3[3]), "f"(nz_2)); - p_3[3] = _fma_571; - float _shfl_xor_sync_573 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 4); - float _add_572; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_572) : "f"(p_3[0]), "f"(_shfl_xor_sync_573)); - p_3[0] = _add_572; - float _shfl_xor_sync_575 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 4); - float _add_574; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(p_3[1]), "f"(_shfl_xor_sync_575)); - p_3[1] = _add_574; - float _shfl_xor_sync_577 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 4); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(p_3[2]), "f"(_shfl_xor_sync_577)); - p_3[2] = _add_576; - float _shfl_xor_sync_579 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 4); - float _add_578; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_578) : "f"(p_3[3]), "f"(_shfl_xor_sync_579)); - p_3[3] = _add_578; - float _shfl_xor_sync_581 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 2); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(p_3[0]), "f"(_shfl_xor_sync_581)); - p_3[0] = _add_580; - float _shfl_xor_sync_583 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 2); - float _add_582; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_582) : "f"(p_3[1]), "f"(_shfl_xor_sync_583)); - p_3[1] = _add_582; - float _shfl_xor_sync_585 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 2); - float _add_584; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(p_3[2]), "f"(_shfl_xor_sync_585)); - p_3[2] = _add_584; - float _shfl_xor_sync_587 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 2); - float _add_586; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_586) : "f"(p_3[3]), "f"(_shfl_xor_sync_587)); - p_3[3] = _add_586; - float _shfl_xor_sync_589 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 1); - float _add_588; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_588) : "f"(p_3[0]), "f"(_shfl_xor_sync_589)); - p_3[0] = _add_588; - float _shfl_xor_sync_591 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 1); - float _add_590; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_590) : "f"(p_3[1]), "f"(_shfl_xor_sync_591)); - p_3[1] = _add_590; - float _shfl_xor_sync_593 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 1); - float _add_592; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_592) : "f"(p_3[2]), "f"(_shfl_xor_sync_593)); - p_3[2] = _add_592; - float _shfl_xor_sync_595 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 1); - float _add_594; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_594) : "f"(p_3[3]), "f"(_shfl_xor_sync_595)); - p_3[3] = _add_594; - float _add_596; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_596) : "f"(p_3[0]), "f"(p_3[2])); - float _add_597; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_597) : "f"(p_3[1]), "f"(p_3[3])); + float _fma_603; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_603) : "f"(gq_3[3]), "f"(y_3[3]), "f"(nz_2)); + p_3[3] = _fma_603; + float _shfl_xor_sync_605 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 4); + float _add_604; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_604) : "f"(p_3[0]), "f"(_shfl_xor_sync_605)); + p_3[0] = _add_604; + float _shfl_xor_sync_607 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 4); + float _add_606; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_606) : "f"(p_3[1]), "f"(_shfl_xor_sync_607)); + p_3[1] = _add_606; + float _shfl_xor_sync_609 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 4); + float _add_608; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_608) : "f"(p_3[2]), "f"(_shfl_xor_sync_609)); + p_3[2] = _add_608; + float _shfl_xor_sync_611 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 4); + float _add_610; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_610) : "f"(p_3[3]), "f"(_shfl_xor_sync_611)); + p_3[3] = _add_610; + float _shfl_xor_sync_613 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 2); + float _add_612; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_612) : "f"(p_3[0]), "f"(_shfl_xor_sync_613)); + p_3[0] = _add_612; + float _shfl_xor_sync_615 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 2); + float _add_614; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_614) : "f"(p_3[1]), "f"(_shfl_xor_sync_615)); + p_3[1] = _add_614; + float _shfl_xor_sync_617 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 2); + float _add_616; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_616) : "f"(p_3[2]), "f"(_shfl_xor_sync_617)); + p_3[2] = _add_616; + float _shfl_xor_sync_619 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 2); + float _add_618; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_618) : "f"(p_3[3]), "f"(_shfl_xor_sync_619)); + p_3[3] = _add_618; + float _shfl_xor_sync_621 = __shfl_xor_sync(0xFFFFFFFF, p_3[0], 1); + float _add_620; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_620) : "f"(p_3[0]), "f"(_shfl_xor_sync_621)); + p_3[0] = _add_620; + float _shfl_xor_sync_623 = __shfl_xor_sync(0xFFFFFFFF, p_3[1], 1); + float _add_622; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_622) : "f"(p_3[1]), "f"(_shfl_xor_sync_623)); + p_3[1] = _add_622; + float _shfl_xor_sync_625 = __shfl_xor_sync(0xFFFFFFFF, p_3[2], 1); + float _add_624; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_624) : "f"(p_3[2]), "f"(_shfl_xor_sync_625)); + p_3[2] = _add_624; + float _shfl_xor_sync_627 = __shfl_xor_sync(0xFFFFFFFF, p_3[3], 1); + float _add_626; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_626) : "f"(p_3[3]), "f"(_shfl_xor_sync_627)); + p_3[3] = _add_626; + float _add_628; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_628) : "f"(p_3[0]), "f"(p_3[2])); + float _add_629; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_629) : "f"(p_3[1]), "f"(p_3[3])); float sq_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3) : "f"(_add_596), "f"(_add_597)); - float _shfl_sync_600 = __shfl_sync(0xFFFFFFFF, sq_3, 0); - float _shfl_sync_601 = __shfl_sync(0xFFFFFFFF, sq_3, 8); - float _add_599; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_599) : "f"(_shfl_sync_600), "f"(_shfl_sync_601)); - float _shfl_sync_602 = __shfl_sync(0xFFFFFFFF, sq_3, 16); - float _add_598; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_598) : "f"(_add_599), "f"(_shfl_sync_602)); - float _shfl_sync_603 = __shfl_sync(0xFFFFFFFF, sq_3, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3) : "f"(_add_628), "f"(_add_629)); + float _shfl_sync_632 = __shfl_sync(0xFFFFFFFF, sq_3, 0); + float _shfl_sync_633 = __shfl_sync(0xFFFFFFFF, sq_3, 8); + float _add_631; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_631) : "f"(_shfl_sync_632), "f"(_shfl_sync_633)); + float _shfl_sync_634 = __shfl_sync(0xFFFFFFFF, sq_3, 16); + float _add_630; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_630) : "f"(_add_631), "f"(_shfl_sync_634)); + float _shfl_sync_635 = __shfl_sync(0xFFFFFFFF, sq_3, 24); float dot_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3) : "f"(_add_598), "f"(_shfl_sync_603)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3) : "f"(_add_630), "f"(_shfl_sync_635)); float o_3[4]; - float _fma_605; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_605) : "f"(y_3[0]), "f"(dot_3), "f"(nz_2)); - float _sub_604; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_604) : "f"(gq_3[0]), "f"(_fma_605)); - o_3[0] = mul_f32_rn_noftz(rq_3[0], _sub_604); - float _fma_607; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_607) : "f"(y_3[1]), "f"(dot_3), "f"(nz_2)); - float _sub_606; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_606) : "f"(gq_3[1]), "f"(_fma_607)); - o_3[1] = mul_f32_rn_noftz(rq_3[0], _sub_606); - float _fma_609; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_609) : "f"(y_3[2]), "f"(dot_3), "f"(nz_2)); - float _sub_608; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_608) : "f"(gq_3[2]), "f"(_fma_609)); - o_3[2] = mul_f32_rn_noftz(rq_3[0], _sub_608); - float _fma_611; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_611) : "f"(y_3[3]), "f"(dot_3), "f"(nz_2)); - float _sub_610; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_610) : "f"(gq_3[3]), "f"(_fma_611)); - o_3[3] = mul_f32_rn_noftz(rq_3[0], _sub_610); + float _fma_637; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_637) : "f"(y_3[0]), "f"(dot_3), "f"(nz_2)); + float _sub_636; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_636) : "f"(gq_3[0]), "f"(_fma_637)); + o_3[0] = mul_f32_rn_noftz(rq_3[0], _sub_636); + float _fma_639; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_639) : "f"(y_3[1]), "f"(dot_3), "f"(nz_2)); + float _sub_638; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_638) : "f"(gq_3[1]), "f"(_fma_639)); + o_3[1] = mul_f32_rn_noftz(rq_3[0], _sub_638); + float _fma_641; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_641) : "f"(y_3[2]), "f"(dot_3), "f"(nz_2)); + float _sub_640; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_640) : "f"(gq_3[2]), "f"(_fma_641)); + o_3[2] = mul_f32_rn_noftz(rq_3[0], _sub_640); + float _fma_643; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_643) : "f"(y_3[3]), "f"(dot_3), "f"(nz_2)); + float _sub_642; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_642) : "f"(gq_3[3]), "f"(_fma_643)); + o_3[3] = mul_f32_rn_noftz(rq_3[0], _sub_642); unsigned int ob_3[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -5755,106 +6033,106 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float y_3_1[4]; float p_3_1[4]; y_3_1[0] = mul_f32_rn_noftz(xk_3[0], rk_3[0]); - float _fma_612; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_612) : "f"(gk_3[0]), "f"(y_3_1[0]), "f"(nz_2)); - p_3_1[0] = _fma_612; + float _fma_644; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_644) : "f"(gk_3[0]), "f"(y_3_1[0]), "f"(nz_2)); + p_3_1[0] = _fma_644; y_3_1[1] = mul_f32_rn_noftz(xk_3[1], rk_3[0]); - float _fma_613; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_613) : "f"(gk_3[1]), "f"(y_3_1[1]), "f"(nz_2)); - p_3_1[1] = _fma_613; + float _fma_645; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_645) : "f"(gk_3[1]), "f"(y_3_1[1]), "f"(nz_2)); + p_3_1[1] = _fma_645; y_3_1[2] = mul_f32_rn_noftz(xk_3[2], rk_3[0]); - float _fma_614; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_614) : "f"(gk_3[2]), "f"(y_3_1[2]), "f"(nz_2)); - p_3_1[2] = _fma_614; + float _fma_646; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_646) : "f"(gk_3[2]), "f"(y_3_1[2]), "f"(nz_2)); + p_3_1[2] = _fma_646; y_3_1[3] = mul_f32_rn_noftz(xk_3[3], rk_3[0]); - float _fma_615; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_615) : "f"(gk_3[3]), "f"(y_3_1[3]), "f"(nz_2)); - p_3_1[3] = _fma_615; - float _shfl_xor_sync_617 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 4); - float _add_616; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_616) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_617)); - p_3_1[0] = _add_616; - float _shfl_xor_sync_619 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 4); - float _add_618; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_618) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_619)); - p_3_1[1] = _add_618; - float _shfl_xor_sync_621 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 4); - float _add_620; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_620) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_621)); - p_3_1[2] = _add_620; - float _shfl_xor_sync_623 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 4); - float _add_622; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_622) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_623)); - p_3_1[3] = _add_622; - float _shfl_xor_sync_625 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 2); - float _add_624; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_624) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_625)); - p_3_1[0] = _add_624; - float _shfl_xor_sync_627 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 2); - float _add_626; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_626) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_627)); - p_3_1[1] = _add_626; - float _shfl_xor_sync_629 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 2); - float _add_628; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_628) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_629)); - p_3_1[2] = _add_628; - float _shfl_xor_sync_631 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 2); - float _add_630; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_630) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_631)); - p_3_1[3] = _add_630; - float _shfl_xor_sync_633 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 1); - float _add_632; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_632) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_633)); - p_3_1[0] = _add_632; - float _shfl_xor_sync_635 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 1); - float _add_634; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_634) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_635)); - p_3_1[1] = _add_634; - float _shfl_xor_sync_637 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 1); - float _add_636; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_636) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_637)); - p_3_1[2] = _add_636; - float _shfl_xor_sync_639 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 1); - float _add_638; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_638) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_639)); - p_3_1[3] = _add_638; - float _add_640; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_640) : "f"(p_3_1[0]), "f"(p_3_1[2])); - float _add_641; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_641) : "f"(p_3_1[1]), "f"(p_3_1[3])); + float _fma_647; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_647) : "f"(gk_3[3]), "f"(y_3_1[3]), "f"(nz_2)); + p_3_1[3] = _fma_647; + float _shfl_xor_sync_649 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 4); + float _add_648; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_648) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_649)); + p_3_1[0] = _add_648; + float _shfl_xor_sync_651 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 4); + float _add_650; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_650) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_651)); + p_3_1[1] = _add_650; + float _shfl_xor_sync_653 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 4); + float _add_652; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_652) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_653)); + p_3_1[2] = _add_652; + float _shfl_xor_sync_655 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 4); + float _add_654; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_654) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_655)); + p_3_1[3] = _add_654; + float _shfl_xor_sync_657 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 2); + float _add_656; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_656) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_657)); + p_3_1[0] = _add_656; + float _shfl_xor_sync_659 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 2); + float _add_658; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_658) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_659)); + p_3_1[1] = _add_658; + float _shfl_xor_sync_661 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 2); + float _add_660; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_660) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_661)); + p_3_1[2] = _add_660; + float _shfl_xor_sync_663 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 2); + float _add_662; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_662) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_663)); + p_3_1[3] = _add_662; + float _shfl_xor_sync_665 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[0], 1); + float _add_664; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_664) : "f"(p_3_1[0]), "f"(_shfl_xor_sync_665)); + p_3_1[0] = _add_664; + float _shfl_xor_sync_667 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[1], 1); + float _add_666; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_666) : "f"(p_3_1[1]), "f"(_shfl_xor_sync_667)); + p_3_1[1] = _add_666; + float _shfl_xor_sync_669 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[2], 1); + float _add_668; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_668) : "f"(p_3_1[2]), "f"(_shfl_xor_sync_669)); + p_3_1[2] = _add_668; + float _shfl_xor_sync_671 = __shfl_xor_sync(0xFFFFFFFF, p_3_1[3], 1); + float _add_670; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_670) : "f"(p_3_1[3]), "f"(_shfl_xor_sync_671)); + p_3_1[3] = _add_670; + float _add_672; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_672) : "f"(p_3_1[0]), "f"(p_3_1[2])); + float _add_673; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_673) : "f"(p_3_1[1]), "f"(p_3_1[3])); float sq_3_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3_1) : "f"(_add_640), "f"(_add_641)); - float _shfl_sync_644 = __shfl_sync(0xFFFFFFFF, sq_3_1, 0); - float _shfl_sync_645 = __shfl_sync(0xFFFFFFFF, sq_3_1, 8); - float _add_643; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_643) : "f"(_shfl_sync_644), "f"(_shfl_sync_645)); - float _shfl_sync_646 = __shfl_sync(0xFFFFFFFF, sq_3_1, 16); - float _add_642; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_642) : "f"(_add_643), "f"(_shfl_sync_646)); - float _shfl_sync_647 = __shfl_sync(0xFFFFFFFF, sq_3_1, 24); + asm("add.rn.f32 %0, %1, %2;" : "=f"(sq_3_1) : "f"(_add_672), "f"(_add_673)); + float _shfl_sync_676 = __shfl_sync(0xFFFFFFFF, sq_3_1, 0); + float _shfl_sync_677 = __shfl_sync(0xFFFFFFFF, sq_3_1, 8); + float _add_675; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_675) : "f"(_shfl_sync_676), "f"(_shfl_sync_677)); + float _shfl_sync_678 = __shfl_sync(0xFFFFFFFF, sq_3_1, 16); + float _add_674; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_674) : "f"(_add_675), "f"(_shfl_sync_678)); + float _shfl_sync_679 = __shfl_sync(0xFFFFFFFF, sq_3_1, 24); float dot_3_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3_1) : "f"(_add_642), "f"(_shfl_sync_647)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(dot_3_1) : "f"(_add_674), "f"(_shfl_sync_679)); float o_3_1[4]; - float _fma_649; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_649) : "f"(y_3_1[0]), "f"(dot_3_1), "f"(nz_2)); - float _sub_648; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_648) : "f"(gk_3[0]), "f"(_fma_649)); - o_3_1[0] = mul_f32_rn_noftz(rk_3[0], _sub_648); - float _fma_651; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_651) : "f"(y_3_1[1]), "f"(dot_3_1), "f"(nz_2)); - float _sub_650; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_650) : "f"(gk_3[1]), "f"(_fma_651)); - o_3_1[1] = mul_f32_rn_noftz(rk_3[0], _sub_650); - float _fma_653; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_653) : "f"(y_3_1[2]), "f"(dot_3_1), "f"(nz_2)); - float _sub_652; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_652) : "f"(gk_3[2]), "f"(_fma_653)); - o_3_1[2] = mul_f32_rn_noftz(rk_3[0], _sub_652); - float _fma_655; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_655) : "f"(y_3_1[3]), "f"(dot_3_1), "f"(nz_2)); - float _sub_654; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_654) : "f"(gk_3[3]), "f"(_fma_655)); - o_3_1[3] = mul_f32_rn_noftz(rk_3[0], _sub_654); + float _fma_681; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_681) : "f"(y_3_1[0]), "f"(dot_3_1), "f"(nz_2)); + float _sub_680; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_680) : "f"(gk_3[0]), "f"(_fma_681)); + o_3_1[0] = mul_f32_rn_noftz(rk_3[0], _sub_680); + float _fma_683; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_683) : "f"(y_3_1[1]), "f"(dot_3_1), "f"(nz_2)); + float _sub_682; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_682) : "f"(gk_3[1]), "f"(_fma_683)); + o_3_1[1] = mul_f32_rn_noftz(rk_3[0], _sub_682); + float _fma_685; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_685) : "f"(y_3_1[2]), "f"(dot_3_1), "f"(nz_2)); + float _sub_684; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_684) : "f"(gk_3[2]), "f"(_fma_685)); + o_3_1[2] = mul_f32_rn_noftz(rk_3[0], _sub_684); + float _fma_687; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_687) : "f"(y_3_1[3]), "f"(dot_3_1), "f"(nz_2)); + float _sub_686; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_686) : "f"(gk_3[3]), "f"(_fma_687)); + o_3_1[3] = mul_f32_rn_noftz(rk_3[0], _sub_686); unsigned int ob_3_1[2]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { @@ -5865,8 +6143,8 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM } asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_656 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_656), "f"(tot_1693) : "memory"); + uint32_t _addr_688 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_688), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5895,9 +6173,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_16 = offset_1710 + _tmem_load_29[0]; if (t_49 < clen) { { - auto _gst_657 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_657)); - *(reinterpret_cast(_gst_657) + (0)) = dg_16; + auto _gst_689 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_689)); + *(reinterpret_cast(_gst_689) + (0)) = dg_16; } } int t_50 = 8 * th + 1; @@ -5905,9 +6183,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_17 = offset_1710 + _tmem_load_29[1]; if (t_50 < clen) { { - auto _gst_658 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_658)); - *(reinterpret_cast(_gst_658) + (0)) = dg_17; + auto _gst_690 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_690)); + *(reinterpret_cast(_gst_690) + (0)) = dg_17; } } int t_51 = 8 * th + 2; @@ -5915,9 +6193,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_18 = offset_1710 + _tmem_load_29[2]; if (t_51 < clen) { { - auto _gst_659 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_659)); - *(reinterpret_cast(_gst_659) + (0)) = dg_18; + auto _gst_691 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_691)); + *(reinterpret_cast(_gst_691) + (0)) = dg_18; } } int t_52 = 8 * th + 3; @@ -5925,9 +6203,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_19 = offset_1710 + _tmem_load_29[3]; if (t_52 < clen) { { - auto _gst_660 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_660)); - *(reinterpret_cast(_gst_660) + (0)) = dg_19; + auto _gst_692 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_692)); + *(reinterpret_cast(_gst_692) + (0)) = dg_19; } } int t_53 = 8 * th + 4; @@ -5935,9 +6213,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_20 = offset_1710 + _tmem_load_29[4]; if (t_53 < clen) { { - auto _gst_661 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_661)); - *(reinterpret_cast(_gst_661) + (0)) = dg_20; + auto _gst_693 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_693)); + *(reinterpret_cast(_gst_693) + (0)) = dg_20; } } int t_54 = 8 * th + 5; @@ -5945,9 +6223,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_21 = offset_1710 + _tmem_load_29[5]; if (t_54 < clen) { { - auto _gst_662 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_662)); - *(reinterpret_cast(_gst_662) + (0)) = dg_21; + auto _gst_694 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_694)); + *(reinterpret_cast(_gst_694) + (0)) = dg_21; } } int t_55 = 8 * th + 6; @@ -5955,9 +6233,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_22 = offset_1710 + _tmem_load_29[6]; if (t_55 < clen) { { - auto _gst_663 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_663)); - *(reinterpret_cast(_gst_663) + (0)) = dg_22; + auto _gst_695 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_695)); + *(reinterpret_cast(_gst_695) + (0)) = dg_22; } } int t_56 = 8 * th + 7; @@ -5965,9 +6243,9 @@ kernel_cake_kda_det_train_328afe43951979e798d1(const __grid_constant__ CUtensorM float dg_23 = offset_1710 + _tmem_load_29[7]; if (t_56 < clen) { { - auto _gst_664 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_664)); - *(reinterpret_cast(_gst_664) + (0)) = dg_23; + auto _gst_696 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_696)); + *(reinterpret_cast(_gst_696) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1708 + t_lo_1709; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu index cff9caa322a..2b8d1329c73 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_b16a0c1729083004fe80_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -199,6 +200,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2267,12 +2284,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_0); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_0 : __fmaf_rn(dl, _rcp_170, e2_0)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2296,12 +2319,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_1); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_1, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_1, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_1 : __fmaf_rn(dl_1, _rcp_174, e2_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2325,12 +2354,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_2); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_2, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_2, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_2 : __fmaf_rn(dl_3, _rcp_178, e2_2)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2354,12 +2389,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_3); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_3, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_3, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_3 : __fmaf_rn(dl_5, _rcp_182, e2_3)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2395,12 +2436,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2420,12 +2467,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2445,12 +2498,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2470,12 +2529,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2523,12 +2588,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2552,12 +2623,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_1 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2581,12 +2658,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2610,12 +2693,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2657,12 +2746,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2686,12 +2781,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2715,12 +2816,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2744,12 +2851,18 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2820,37 +2933,37 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float2 d2_56 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_56.x); float e1_12 = approx_exp2(d2_56.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_57.x); float e1_13 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_58.x); float e1_14 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_59.x); float e1_15 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2862,29 +2975,29 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2892,29 +3005,29 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2984,7 +3097,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3016,7 +3134,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3048,7 +3171,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_1)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3080,7 +3208,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_2)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3112,7 +3245,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_3)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3144,7 +3282,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_4)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3176,7 +3319,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_5)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3208,7 +3356,12 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_6)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3223,31 +3376,31 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_216 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_216; + float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_248; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_217 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_217; + float _shfl_xor_sync_249 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_249; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_218 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_218; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_250; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_219 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_219; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_251; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_220 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_220; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_252; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_221 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_221; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_253; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_222 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_222; - float _shfl_xor_sync_223 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_223; - float _shfl_xor_sync_224 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_224; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_254; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_255; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_256; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); if (32 + 8 * th < clen) { @@ -3308,322 +3461,135 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float r_272[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1049 = r_272[0]; - float gc_1050 = gn_1049 - 21.640425f; - int eb_1051 = 98304 + th * -32768; - int ew_1052 = eb_1051 >> 2; - int qbase_1053 = pool_f32_addr + (unsigned int)(eb_1051 + 12288 + dh_lane * 1024); - int sb8_1054 = 48 + 8 * th; - int gb8_1055 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1054 * 128); - int kb8_1056 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1054 * 128); - float dbp_1057[8]; - float _tmem_load_12[8]; - tmem_ld_x8(&_tmem_load_12[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float _tmem_load_13[8]; - tmem_ld_x8(&_tmem_load_13[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float _tmem_load_14[8]; - tmem_ld_x8(&_tmem_load_14[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float _tmem_load_15[8]; - tmem_ld_x8(&_tmem_load_15[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float _tmem_load_16[8]; - tmem_ld_x8(&_tmem_load_16[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_1058[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[0])), "=r"(*reinterpret_cast(&btv_1058[(0) + 1])), "=r"(*reinterpret_cast(&btv_1058[(0) + 2])), "=r"(*reinterpret_cast(&btv_1058[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[4])), "=r"(*reinterpret_cast(&btv_1058[(4) + 1])), "=r"(*reinterpret_cast(&btv_1058[(4) + 2])), "=r"(*reinterpret_cast(&btv_1058[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); + float gn_0 = r_272[0]; + float gc_0 = gn_0 - 21.640425f; + int eb_0 = 98304 + th * -32768; + int ew_0 = eb_0 >> 2; + int qbase_0 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); + int sb8_0 = 48 + 8 * th; + int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); + int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); + int c_qa = 160 + 8 * th; + int c_ka = 176 + 8 * th; + int c_qd = 224 + 8 * th; + int c_kd = 240 + 8 * th; + int c_dk = 48 + 8 * th; + int c_bp = c_ka; + float s_run_0[1]; + s_run_0[0] = 0.0f; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j = 7 - k; + float a_qd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j))); + float a_kd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd + j))); + float a_dk[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk + j))); + float a_qa[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa + j))); + float a_ka[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka + j))); + float btr[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_0 + j)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8_0 + j; + float r_273[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_273[0]; + unsigned int r_274[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_274[0] << 16); + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_275[0] << 16); + float q_m = ((t >= clen) ? 0.0f : q_t); + float dqf = pool_f32[ew_0 + 128 * j + d_lane]; + float dkf = pool_f32[ew_0 + (1024 + 128 * j) + d_lane]; + float dgf = pool_f32[ew_0 + (2048 + 128 * j) + d_lane]; + float bt_t = btr[0]; + float e_dg = approx_exp2(g_t - gc_0); + float e_off = approx_exp2(g_t - gn_0); + float dq2 = __fmaf_rn(a_qa[0], e_off, a_qd[0] * e_dg); + float dk2 = __fmaf_rn(a_ka[0], e_off, a_kd[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + a_dk[0]; + float qe_7 = q_m * e_dg; + float kp_19 = k_t * bt_t; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, a_qd[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg) * a_kd[0]); + float dg_o = __fmaf_rn(q_m, dq2, __fmaf_rn(dk2b - a_dk[0], k_t, cr_7)) + dgf; + s_run_0[0] = s_run_0[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(98304 - 32768 * th >> 2) + (128 * j + d_lane)] = dq_o; + pool_f32[(98304 - 32768 * th >> 2) + (1024 + 128 * j + d_lane)] = dk_o; + } + float tot_0 = s_run_0[0]; + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1059[1]; - s_run_1059[0] = 0.0f; - int t_0 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew_1052 + 896 + d_lane]; - float dkf_0 = pool_f32[ew_1052 + 1920 + d_lane]; - float dgf_0 = pool_f32[ew_1052 + 2944 + d_lane]; - float bt_t_0 = btv_1058[7]; - float e_off_0 = approx_exp2(g_t_0 - gn_1049); - float e_dg_0 = approx_exp2(g_t_0 - gc_1050); - float dq2_0 = __fmaf_rn(_tmem_load_12[7], e_off_0, _tmem_load_14[7] * e_dg_0); - float dk2_0 = __fmaf_rn(_tmem_load_13[7], e_off_0, _tmem_load_15[7] * e_dg_0); - dbp_1057[7] = dk2_0 * k_t_0; - float dk2b_0 = dk2_0 * bt_t_0; - float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; - s_run_1059[0] = s_run_1059[0] + dg_o_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_0; - pool_f32[(98304 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_0; - int t_1 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew_1052 + 768 + d_lane]; - float dkf_1 = pool_f32[ew_1052 + 1792 + d_lane]; - float dgf_1 = pool_f32[ew_1052 + 2816 + d_lane]; - float bt_t_1 = btv_1058[6]; - float e_off_1 = approx_exp2(g_t_1 - gn_1049); - float e_dg_1 = approx_exp2(g_t_1 - gc_1050); - float dq2_1 = __fmaf_rn(_tmem_load_12[6], e_off_1, _tmem_load_14[6] * e_dg_1); - float dk2_1 = __fmaf_rn(_tmem_load_13[6], e_off_1, _tmem_load_15[6] * e_dg_1); - dbp_1057[6] = dk2_1 * k_t_1; - float dk2b_1 = dk2_1 * bt_t_1; - float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + _tmem_load_16[6]; - float dg_o_1 = __fmaf_rn(q_t_1_9, dq2_1, (dk2b_1 - _tmem_load_16[6]) * k_t_1) + dgf_1; - s_run_1059[0] = s_run_1059[0] + dg_o_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_1; - pool_f32[(98304 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_1; - int t_2 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew_1052 + 640 + d_lane]; - float dkf_2 = pool_f32[ew_1052 + 1664 + d_lane]; - float dgf_2 = pool_f32[ew_1052 + 2688 + d_lane]; - float bt_t_2 = btv_1058[5]; - float e_off_2 = approx_exp2(g_t_2 - gn_1049); - float e_dg_2 = approx_exp2(g_t_2 - gc_1050); - float dq2_2 = __fmaf_rn(_tmem_load_12[5], e_off_2, _tmem_load_14[5] * e_dg_2); - float dk2_2 = __fmaf_rn(_tmem_load_13[5], e_off_2, _tmem_load_15[5] * e_dg_2); - dbp_1057[5] = dk2_2 * k_t_2; - float dk2b_2 = dk2_2 * bt_t_2; - float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[5]; - float dg_o_2 = __fmaf_rn(q_t_1_10, dq2_2, (dk2b_2 - _tmem_load_16[5]) * k_t_2) + dgf_2; - s_run_1059[0] = s_run_1059[0] + dg_o_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_2; - pool_f32[(98304 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_2; - int t_3 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew_1052 + 512 + d_lane]; - float dkf_3 = pool_f32[ew_1052 + 1536 + d_lane]; - float dgf_3 = pool_f32[ew_1052 + 2560 + d_lane]; - float bt_t_3 = btv_1058[4]; - float e_off_3 = approx_exp2(g_t_3 - gn_1049); - float e_dg_3 = approx_exp2(g_t_3 - gc_1050); - float dq2_3 = __fmaf_rn(_tmem_load_12[4], e_off_3, _tmem_load_14[4] * e_dg_3); - float dk2_3 = __fmaf_rn(_tmem_load_13[4], e_off_3, _tmem_load_15[4] * e_dg_3); - dbp_1057[4] = dk2_3 * k_t_3; - float dk2b_3 = dk2_3 * bt_t_3; - float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[4]; - float dg_o_3 = __fmaf_rn(q_t_1_11, dq2_3, (dk2b_3 - _tmem_load_16[4]) * k_t_3) + dgf_3; - s_run_1059[0] = s_run_1059[0] + dg_o_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_3; - pool_f32[(98304 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_3; - int t_4 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew_1052 + 384 + d_lane]; - float dkf_4 = pool_f32[ew_1052 + 1408 + d_lane]; - float dgf_4 = pool_f32[ew_1052 + 2432 + d_lane]; - float bt_t_4 = btv_1058[3]; - float e_off_4 = approx_exp2(g_t_4 - gn_1049); - float e_dg_4 = approx_exp2(g_t_4 - gc_1050); - float dq2_4 = __fmaf_rn(_tmem_load_12[3], e_off_4, _tmem_load_14[3] * e_dg_4); - float dk2_4 = __fmaf_rn(_tmem_load_13[3], e_off_4, _tmem_load_15[3] * e_dg_4); - dbp_1057[3] = dk2_4 * k_t_4; - float dk2b_4 = dk2_4 * bt_t_4; - float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[3]; - float dg_o_4 = __fmaf_rn(q_t_1_12, dq2_4, (dk2b_4 - _tmem_load_16[3]) * k_t_4) + dgf_4; - s_run_1059[0] = s_run_1059[0] + dg_o_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_4; - pool_f32[(98304 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_4; - int t_5 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew_1052 + 256 + d_lane]; - float dkf_5 = pool_f32[ew_1052 + 1280 + d_lane]; - float dgf_5 = pool_f32[ew_1052 + 2304 + d_lane]; - float bt_t_5 = btv_1058[2]; - float e_off_5 = approx_exp2(g_t_5 - gn_1049); - float e_dg_5 = approx_exp2(g_t_5 - gc_1050); - float dq2_5 = __fmaf_rn(_tmem_load_12[2], e_off_5, _tmem_load_14[2] * e_dg_5); - float dk2_5 = __fmaf_rn(_tmem_load_13[2], e_off_5, _tmem_load_15[2] * e_dg_5); - dbp_1057[2] = dk2_5 * k_t_5; - float dk2b_5 = dk2_5 * bt_t_5; - float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[2]; - float dg_o_5 = __fmaf_rn(q_t_1_13, dq2_5, (dk2b_5 - _tmem_load_16[2]) * k_t_5) + dgf_5; - s_run_1059[0] = s_run_1059[0] + dg_o_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_5; - pool_f32[(98304 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_5; - int t_6 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew_1052 + 128 + d_lane]; - float dkf_6 = pool_f32[ew_1052 + 1152 + d_lane]; - float dgf_6 = pool_f32[ew_1052 + 2176 + d_lane]; - float bt_t_6 = btv_1058[1]; - float e_off_6 = approx_exp2(g_t_6 - gn_1049); - float e_dg_6 = approx_exp2(g_t_6 - gc_1050); - float dq2_6 = __fmaf_rn(_tmem_load_12[1], e_off_6, _tmem_load_14[1] * e_dg_6); - float dk2_6 = __fmaf_rn(_tmem_load_13[1], e_off_6, _tmem_load_15[1] * e_dg_6); - dbp_1057[1] = dk2_6 * k_t_6; - float dk2b_6 = dk2_6 * bt_t_6; - float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[1]; - float dg_o_6 = __fmaf_rn(q_t_1_14, dq2_6, (dk2b_6 - _tmem_load_16[1]) * k_t_6) + dgf_6; - s_run_1059[0] = s_run_1059[0] + dg_o_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_6; - pool_f32[(98304 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_6; - int t_7 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew_1052 + d_lane]; - float dkf_7 = pool_f32[ew_1052 + 1024 + d_lane]; - float dgf_7 = pool_f32[ew_1052 + 2048 + d_lane]; - float bt_t_7 = btv_1058[0]; - float e_off_7 = approx_exp2(g_t_7 - gn_1049); - float e_dg_7 = approx_exp2(g_t_7 - gc_1050); - float dq2_7 = __fmaf_rn(_tmem_load_12[0], e_off_7, _tmem_load_14[0] * e_dg_7); - float dk2_7 = __fmaf_rn(_tmem_load_13[0], e_off_7, _tmem_load_15[0] * e_dg_7); - dbp_1057[0] = dk2_7 * k_t_7; - float dk2b_7 = dk2_7 * bt_t_7; - float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[0]; - float dg_o_7 = __fmaf_rn(q_t_1_15, dq2_7, (dk2b_7 - _tmem_load_16[0]) * k_t_7) + dgf_7; - s_run_1059[0] = s_run_1059[0] + dg_o_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + d_lane] = dq_o_7; - pool_f32[(98304 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_7; - float tot_1228 = s_run_1059[0]; - int b4_1229 = lane_1 >> 4 & 1; - int b3_1230 = lane_1 >> 3 & 1; - int b2_1231 = lane_1 >> 2 & 1; - float v4_1232[4]; - float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_225 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_225; - float send_1 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_226 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_226; - float send_2 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_227 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_227; - float send_3 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_228 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_228; - float v2_1237[2]; - float send_4 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_229 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_229; - float send_5 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_230 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_230; - float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_231 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_231; - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_232; - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_233; + int b4_0 = lane_1 >> 4 & 1; + int b3_0 = lane_1 >> 3 & 1; + int b2_0 = lane_1 >> 2 & 1; + float v4_0[4]; + float send_0 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float send_1 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float send_2 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); + float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float send_3 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); + float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float v2[2]; + float send2_0 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send2_0, 8); + v2[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float send2_1 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send2_1, 8); + v2[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float send1 = ((b2_0 == 0) ? v2[1] : v2[0]); + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send1, 4); + float v1_0 = ((b2_0 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_263; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_264; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_265; asm volatile("tcgen05.fence::before_thread_sync;"); if (48 + 8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + (48 + 8 * th)) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(98304 - 32768 * th >> 2) + d_lane]; @@ -3723,8 +3689,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; { - uint32_t _addr_234 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_234), "f"(tot_1228) : "memory"); + uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3742,8 +3708,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi = ((th == 1) ? tot_1228 : t_other[0]); - float t_lo = ((th == 1) ? t_other[0] : tot_1228); + float t_hi = ((th == 1) ? tot_0 : t_other[0]); + float t_lo = ((th == 1) ? t_other[0] : tot_0); float offset = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi); float sfx[8]; tmem_ld_x8(&sfx[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); @@ -3756,132 +3722,132 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_235 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_267 = make_float2(gacc[4], gacc[4]); float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_235), "l"(*(const unsigned long long*)&rp_u)); - float _abs_238; - asm("abs.f32 %0, %1;" : "=f"(_abs_238) : "f"(rp_z.x)); - float _neg_237; - asm("neg.f32 %0, %1;" : "=f"(_neg_237) : "f"(_abs_238)); - float _exp_236 = __nv_expf(_neg_237); - float _abs_241; - asm("abs.f32 %0, %1;" : "=f"(_abs_241) : "f"(rp_z.y)); - float _neg_240; - asm("neg.f32 %0, %1;" : "=f"(_neg_240) : "f"(_abs_241)); - float _exp_239 = __nv_expf(_neg_240); - float2 rp_e = make_float2(_exp_236, _exp_239); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&rp_u)); + float _abs_270; + asm("abs.f32 %0, %1;" : "=f"(_abs_270) : "f"(rp_z.x)); + float _neg_269; + asm("neg.f32 %0, %1;" : "=f"(_neg_269) : "f"(_abs_270)); + float _exp_268 = __nv_expf(_neg_269); + float _abs_273; + asm("abs.f32 %0, %1;" : "=f"(_abs_273) : "f"(rp_z.y)); + float _neg_272; + asm("neg.f32 %0, %1;" : "=f"(_neg_272) : "f"(_abs_273)); + float _exp_271 = __nv_expf(_neg_272); + float2 rp_e = make_float2(_exp_268, _exp_271); float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_242 = approx_rcp(rp_den.x); - float _rcp_243 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_244 = make_float2(_rcp_242, _rcp_243); + float _rcp_274 = approx_rcp(rp_den.x); + float _rcp_275 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_276 = make_float2(_rcp_274, _rcp_275); float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_244)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_276)); float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_245 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_277 = make_float2(gacc[5], gacc[5]); float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_245)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_246 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_278 = make_float2(gacc[4], gacc[4]); float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_246), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_249; - asm("abs.f32 %0, %1;" : "=f"(_abs_249) : "f"(rp_z_0.x)); - float _neg_248; - asm("neg.f32 %0, %1;" : "=f"(_neg_248) : "f"(_abs_249)); - float _exp_247 = __nv_expf(_neg_248); - float _abs_252; - asm("abs.f32 %0, %1;" : "=f"(_abs_252) : "f"(rp_z_0.y)); - float _neg_251; - asm("neg.f32 %0, %1;" : "=f"(_neg_251) : "f"(_abs_252)); - float _exp_250 = __nv_expf(_neg_251); - float2 rp_e_0 = make_float2(_exp_247, _exp_250); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_278), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_281; + asm("abs.f32 %0, %1;" : "=f"(_abs_281) : "f"(rp_z_0.x)); + float _neg_280; + asm("neg.f32 %0, %1;" : "=f"(_neg_280) : "f"(_abs_281)); + float _exp_279 = __nv_expf(_neg_280); + float _abs_284; + asm("abs.f32 %0, %1;" : "=f"(_abs_284) : "f"(rp_z_0.y)); + float _neg_283; + asm("neg.f32 %0, %1;" : "=f"(_neg_283) : "f"(_abs_284)); + float _exp_282 = __nv_expf(_neg_283); + float2 rp_e_0 = make_float2(_exp_279, _exp_282); float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_253 = approx_rcp(rp_den_0.x); - float _rcp_254 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_255 = make_float2(_rcp_253, _rcp_254); + float _rcp_285 = approx_rcp(rp_den_0.x); + float _rcp_286 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_287 = make_float2(_rcp_285, _rcp_286); float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_255)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_287)); float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_256 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_288 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_256)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_288)); u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_257 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_289 = make_float2(gacc[4], gacc[4]); float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_257), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_260; - asm("abs.f32 %0, %1;" : "=f"(_abs_260) : "f"(rp_z_1.x)); - float _neg_259; - asm("neg.f32 %0, %1;" : "=f"(_neg_259) : "f"(_abs_260)); - float _exp_258 = __nv_expf(_neg_259); - float _abs_263; - asm("abs.f32 %0, %1;" : "=f"(_abs_263) : "f"(rp_z_1.y)); - float _neg_262; - asm("neg.f32 %0, %1;" : "=f"(_neg_262) : "f"(_abs_263)); - float _exp_261 = __nv_expf(_neg_262); - float2 rp_e_1 = make_float2(_exp_258, _exp_261); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_289), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_292; + asm("abs.f32 %0, %1;" : "=f"(_abs_292) : "f"(rp_z_1.x)); + float _neg_291; + asm("neg.f32 %0, %1;" : "=f"(_neg_291) : "f"(_abs_292)); + float _exp_290 = __nv_expf(_neg_291); + float _abs_295; + asm("abs.f32 %0, %1;" : "=f"(_abs_295) : "f"(rp_z_1.y)); + float _neg_294; + asm("neg.f32 %0, %1;" : "=f"(_neg_294) : "f"(_abs_295)); + float _exp_293 = __nv_expf(_neg_294); + float2 rp_e_1 = make_float2(_exp_290, _exp_293); float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_264 = approx_rcp(rp_den_1.x); - float _rcp_265 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_266 = make_float2(_rcp_264, _rcp_265); + float _rcp_296 = approx_rcp(rp_den_1.x); + float _rcp_297 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_298 = make_float2(_rcp_296, _rcp_297); float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_266)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_298)); float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_267 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_299 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_267)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_299)); u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_268 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_300 = make_float2(gacc[4], gacc[4]); float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_268), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_271; - asm("abs.f32 %0, %1;" : "=f"(_abs_271) : "f"(rp_z_2.x)); - float _neg_270; - asm("neg.f32 %0, %1;" : "=f"(_neg_270) : "f"(_abs_271)); - float _exp_269 = __nv_expf(_neg_270); - float _abs_274; - asm("abs.f32 %0, %1;" : "=f"(_abs_274) : "f"(rp_z_2.y)); - float _neg_273; - asm("neg.f32 %0, %1;" : "=f"(_neg_273) : "f"(_abs_274)); - float _exp_272 = __nv_expf(_neg_273); - float2 rp_e_2 = make_float2(_exp_269, _exp_272); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_303; + asm("abs.f32 %0, %1;" : "=f"(_abs_303) : "f"(rp_z_2.x)); + float _neg_302; + asm("neg.f32 %0, %1;" : "=f"(_neg_302) : "f"(_abs_303)); + float _exp_301 = __nv_expf(_neg_302); + float _abs_306; + asm("abs.f32 %0, %1;" : "=f"(_abs_306) : "f"(rp_z_2.y)); + float _neg_305; + asm("neg.f32 %0, %1;" : "=f"(_neg_305) : "f"(_abs_306)); + float _exp_304 = __nv_expf(_neg_305); + float2 rp_e_2 = make_float2(_exp_301, _exp_304); float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_275 = approx_rcp(rp_den_2.x); - float _rcp_276 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_277 = make_float2(_rcp_275, _rcp_276); + float _rcp_307 = approx_rcp(rp_den_2.x); + float _rcp_308 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_309 = make_float2(_rcp_307, _rcp_308); float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_278 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_310 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_278)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_310)); u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -3896,54 +3862,54 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_279; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_279) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_279; + uint32_t _prmt_311; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_311) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_311; int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_280 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = pq_0[0]; + auto _gst_312 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = pq_0[0]; } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_281; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_281) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_281; + uint32_t _prmt_313; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_313) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_313; int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_282 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = pq_1[0]; + auto _gst_314 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = pq_1[0]; } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_283; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_283) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_283; + uint32_t _prmt_315; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_315; int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_284 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast(_gst_284) + (0)) = pq_2[0]; + auto _gst_316 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = pq_2[0]; } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_285; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_285) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_285; + uint32_t _prmt_317; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_317; int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_286 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast(_gst_286) + (0)) = pq_3[0]; + auto _gst_318 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = pq_3[0]; } } int t_a_0 = 48 + 8 * th; @@ -4008,47 +3974,47 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_287; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_287) : "f"(s_a), "f"(nm_t)); + float _sub_319; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_319) : "f"(s_a), "f"(nm_t)); float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_287), "f"(pa)); - float _sub_288; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_288) : "f"(pa), "f"(nm_t)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_319), "f"(pa)); + float _sub_320; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_320) : "f"(pa), "f"(nm_t)); float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_288), "f"(s_a)); - float _abs_290; - asm("abs.f32 %0, %1;" : "=f"(_abs_290) : "f"(s_a)); - float _abs_291; - asm("abs.f32 %0, %1;" : "=f"(_abs_291) : "f"(pa)); - uint32_t _setp_289_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_289_bits) : "f"(_abs_290), "f"(_abs_291)); - bool _setp_289 = _setp_289_bits != 0; - float nm_err = ((_setp_289) ? nm_es : nm_ex); - float _add_292; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_292) : "f"(c_a), "f"(nm_err)); - c_a = _add_292; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_320), "f"(s_a)); + float _abs_322; + asm("abs.f32 %0, %1;" : "=f"(_abs_322) : "f"(s_a)); + float _abs_323; + asm("abs.f32 %0, %1;" : "=f"(_abs_323) : "f"(pa)); + uint32_t _setp_321_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_321_bits) : "f"(_abs_322), "f"(_abs_323)); + bool _setp_321 = _setp_321_bits != 0; + float nm_err = ((_setp_321) ? nm_es : nm_ex); + float _add_324; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(c_a), "f"(nm_err)); + c_a = _add_324; s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_293; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_293) : "f"(s_b), "f"(nm_t_0)); + float _sub_325; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_325) : "f"(s_b), "f"(nm_t_0)); float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_293), "f"(pb)); - float _sub_294; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_294) : "f"(pb), "f"(nm_t_0)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_325), "f"(pb)); + float _sub_326; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_326) : "f"(pb), "f"(nm_t_0)); float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_294), "f"(s_b)); - float _abs_296; - asm("abs.f32 %0, %1;" : "=f"(_abs_296) : "f"(s_b)); - float _abs_297; - asm("abs.f32 %0, %1;" : "=f"(_abs_297) : "f"(pb)); - uint32_t _setp_295_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_295_bits) : "f"(_abs_296), "f"(_abs_297)); - bool _setp_295 = _setp_295_bits != 0; - float nm_err_0 = ((_setp_295) ? nm_es_0 : nm_ex_0); - float _add_298; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_298) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_298; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_326), "f"(s_b)); + float _abs_328; + asm("abs.f32 %0, %1;" : "=f"(_abs_328) : "f"(s_b)); + float _abs_329; + asm("abs.f32 %0, %1;" : "=f"(_abs_329) : "f"(pb)); + uint32_t _setp_327_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_327_bits) : "f"(_abs_328), "f"(_abs_329)); + bool _setp_327 = _setp_327_bits != 0; + float nm_err_0 = ((_setp_327) ? nm_es_0 : nm_ex_0); + float _add_330; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_330; s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4072,132 +4038,132 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_299 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_331 = make_float2(gacc[4], gacc[4]); float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_299), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_302; - asm("abs.f32 %0, %1;" : "=f"(_abs_302) : "f"(rp_z_3.x)); - float _neg_301; - asm("neg.f32 %0, %1;" : "=f"(_neg_301) : "f"(_abs_302)); - float _exp_300 = __nv_expf(_neg_301); - float _abs_305; - asm("abs.f32 %0, %1;" : "=f"(_abs_305) : "f"(rp_z_3.y)); - float _neg_304; - asm("neg.f32 %0, %1;" : "=f"(_neg_304) : "f"(_abs_305)); - float _exp_303 = __nv_expf(_neg_304); - float2 rp_e_3 = make_float2(_exp_300, _exp_303); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_331), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_334; + asm("abs.f32 %0, %1;" : "=f"(_abs_334) : "f"(rp_z_3.x)); + float _neg_333; + asm("neg.f32 %0, %1;" : "=f"(_neg_333) : "f"(_abs_334)); + float _exp_332 = __nv_expf(_neg_333); + float _abs_337; + asm("abs.f32 %0, %1;" : "=f"(_abs_337) : "f"(rp_z_3.y)); + float _neg_336; + asm("neg.f32 %0, %1;" : "=f"(_neg_336) : "f"(_abs_337)); + float _exp_335 = __nv_expf(_neg_336); + float2 rp_e_3 = make_float2(_exp_332, _exp_335); float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_306 = approx_rcp(rp_den_3.x); - float _rcp_307 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_308 = make_float2(_rcp_306, _rcp_307); + float _rcp_338 = approx_rcp(rp_den_3.x); + float _rcp_339 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_340 = make_float2(_rcp_338, _rcp_339); float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_308)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_340)); float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_309 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_341 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_310 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_342 = make_float2(gacc[4], gacc[4]); float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_310), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_313; - asm("abs.f32 %0, %1;" : "=f"(_abs_313) : "f"(rp_z_4.x)); - float _neg_312; - asm("neg.f32 %0, %1;" : "=f"(_neg_312) : "f"(_abs_313)); - float _exp_311 = __nv_expf(_neg_312); - float _abs_316; - asm("abs.f32 %0, %1;" : "=f"(_abs_316) : "f"(rp_z_4.y)); - float _neg_315; - asm("neg.f32 %0, %1;" : "=f"(_neg_315) : "f"(_abs_316)); - float _exp_314 = __nv_expf(_neg_315); - float2 rp_e_4 = make_float2(_exp_311, _exp_314); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_342), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_345; + asm("abs.f32 %0, %1;" : "=f"(_abs_345) : "f"(rp_z_4.x)); + float _neg_344; + asm("neg.f32 %0, %1;" : "=f"(_neg_344) : "f"(_abs_345)); + float _exp_343 = __nv_expf(_neg_344); + float _abs_348; + asm("abs.f32 %0, %1;" : "=f"(_abs_348) : "f"(rp_z_4.y)); + float _neg_347; + asm("neg.f32 %0, %1;" : "=f"(_neg_347) : "f"(_abs_348)); + float _exp_346 = __nv_expf(_neg_347); + float2 rp_e_4 = make_float2(_exp_343, _exp_346); float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_317 = approx_rcp(rp_den_4.x); - float _rcp_318 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_319 = make_float2(_rcp_317, _rcp_318); + float _rcp_349 = approx_rcp(rp_den_4.x); + float _rcp_350 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_351 = make_float2(_rcp_349, _rcp_350); float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_319)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_351)); float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_320 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_352 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_320)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_352)); u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_321 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_353 = make_float2(gacc[4], gacc[4]); float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_321), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_324; - asm("abs.f32 %0, %1;" : "=f"(_abs_324) : "f"(rp_z_5.x)); - float _neg_323; - asm("neg.f32 %0, %1;" : "=f"(_neg_323) : "f"(_abs_324)); - float _exp_322 = __nv_expf(_neg_323); - float _abs_327; - asm("abs.f32 %0, %1;" : "=f"(_abs_327) : "f"(rp_z_5.y)); - float _neg_326; - asm("neg.f32 %0, %1;" : "=f"(_neg_326) : "f"(_abs_327)); - float _exp_325 = __nv_expf(_neg_326); - float2 rp_e_5 = make_float2(_exp_322, _exp_325); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_356; + asm("abs.f32 %0, %1;" : "=f"(_abs_356) : "f"(rp_z_5.x)); + float _neg_355; + asm("neg.f32 %0, %1;" : "=f"(_neg_355) : "f"(_abs_356)); + float _exp_354 = __nv_expf(_neg_355); + float _abs_359; + asm("abs.f32 %0, %1;" : "=f"(_abs_359) : "f"(rp_z_5.y)); + float _neg_358; + asm("neg.f32 %0, %1;" : "=f"(_neg_358) : "f"(_abs_359)); + float _exp_357 = __nv_expf(_neg_358); + float2 rp_e_5 = make_float2(_exp_354, _exp_357); float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_328 = approx_rcp(rp_den_5.x); - float _rcp_329 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_330 = make_float2(_rcp_328, _rcp_329); + float _rcp_360 = approx_rcp(rp_den_5.x); + float _rcp_361 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_362 = make_float2(_rcp_360, _rcp_361); float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_330)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_362)); float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_331 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_363 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_331)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_363)); u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_332 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_364 = make_float2(gacc[4], gacc[4]); float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_332), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_335; - asm("abs.f32 %0, %1;" : "=f"(_abs_335) : "f"(rp_z_6.x)); - float _neg_334; - asm("neg.f32 %0, %1;" : "=f"(_neg_334) : "f"(_abs_335)); - float _exp_333 = __nv_expf(_neg_334); - float _abs_338; - asm("abs.f32 %0, %1;" : "=f"(_abs_338) : "f"(rp_z_6.y)); - float _neg_337; - asm("neg.f32 %0, %1;" : "=f"(_neg_337) : "f"(_abs_338)); - float _exp_336 = __nv_expf(_neg_337); - float2 rp_e_6 = make_float2(_exp_333, _exp_336); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_364), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_367; + asm("abs.f32 %0, %1;" : "=f"(_abs_367) : "f"(rp_z_6.x)); + float _neg_366; + asm("neg.f32 %0, %1;" : "=f"(_neg_366) : "f"(_abs_367)); + float _exp_365 = __nv_expf(_neg_366); + float _abs_370; + asm("abs.f32 %0, %1;" : "=f"(_abs_370) : "f"(rp_z_6.y)); + float _neg_369; + asm("neg.f32 %0, %1;" : "=f"(_neg_369) : "f"(_abs_370)); + float _exp_368 = __nv_expf(_neg_369); + float2 rp_e_6 = make_float2(_exp_365, _exp_368); float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_339 = approx_rcp(rp_den_6.x); - float _rcp_340 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_341 = make_float2(_rcp_339, _rcp_340); + float _rcp_371 = approx_rcp(rp_den_6.x); + float _rcp_372 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_373 = make_float2(_rcp_371, _rcp_372); float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_373)); float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_342 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_374 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_342)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_374)); u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4212,54 +4178,54 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_343; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_343) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_343; + uint32_t _prmt_375; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_375) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_375; int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_344 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_344)); - *(reinterpret_cast(_gst_344) + (0)) = pq_4[0]; + auto _gst_376 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_376)); + *(reinterpret_cast(_gst_376) + (0)) = pq_4[0]; } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_345; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_345) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_345; + uint32_t _prmt_377; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_377) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_377; int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_346 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = pq_5[0]; + auto _gst_378 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = pq_5[0]; } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_347; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_347) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_347; + uint32_t _prmt_379; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_379; int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_348 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = pq_6[0]; + auto _gst_380 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = pq_6[0]; } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_349; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_349) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_349; + uint32_t _prmt_381; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_381; int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_350 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = pq_7[0]; + auto _gst_382 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = pq_7[0]; } } int t_a_8 = 32 + 8 * th; @@ -4324,47 +4290,47 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_351; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_351) : "f"(s_a_0), "f"(nm_t_1)); + float _sub_383; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_383) : "f"(s_a_0), "f"(nm_t_1)); float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_351), "f"(pa_0)); - float _sub_352; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_352) : "f"(pa_0), "f"(nm_t_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_383), "f"(pa_0)); + float _sub_384; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_384) : "f"(pa_0), "f"(nm_t_1)); float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_352), "f"(s_a_0)); - float _abs_354; - asm("abs.f32 %0, %1;" : "=f"(_abs_354) : "f"(s_a_0)); - float _abs_355; - asm("abs.f32 %0, %1;" : "=f"(_abs_355) : "f"(pa_0)); - uint32_t _setp_353_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_353_bits) : "f"(_abs_354), "f"(_abs_355)); - bool _setp_353 = _setp_353_bits != 0; - float nm_err_1 = ((_setp_353) ? nm_es_1 : nm_ex_1); - float _add_356; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_356) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_356; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_384), "f"(s_a_0)); + float _abs_386; + asm("abs.f32 %0, %1;" : "=f"(_abs_386) : "f"(s_a_0)); + float _abs_387; + asm("abs.f32 %0, %1;" : "=f"(_abs_387) : "f"(pa_0)); + uint32_t _setp_385_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_385_bits) : "f"(_abs_386), "f"(_abs_387)); + bool _setp_385 = _setp_385_bits != 0; + float nm_err_1 = ((_setp_385) ? nm_es_1 : nm_ex_1); + float _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_388; s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_357; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_357) : "f"(s_b_0), "f"(nm_t_2)); + float _sub_389; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_389) : "f"(s_b_0), "f"(nm_t_2)); float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_357), "f"(pb_0)); - float _sub_358; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_358) : "f"(pb_0), "f"(nm_t_2)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_389), "f"(pb_0)); + float _sub_390; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_390) : "f"(pb_0), "f"(nm_t_2)); float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_358), "f"(s_b_0)); - float _abs_360; - asm("abs.f32 %0, %1;" : "=f"(_abs_360) : "f"(s_b_0)); - float _abs_361; - asm("abs.f32 %0, %1;" : "=f"(_abs_361) : "f"(pb_0)); - uint32_t _setp_359_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_359_bits) : "f"(_abs_360), "f"(_abs_361)); - bool _setp_359 = _setp_359_bits != 0; - float nm_err_2 = ((_setp_359) ? nm_es_2 : nm_ex_2); - float _add_362; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_362) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_362; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_390), "f"(s_b_0)); + float _abs_392; + asm("abs.f32 %0, %1;" : "=f"(_abs_392) : "f"(s_b_0)); + float _abs_393; + asm("abs.f32 %0, %1;" : "=f"(_abs_393) : "f"(pb_0)); + uint32_t _setp_391_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_391_bits) : "f"(_abs_392), "f"(_abs_393)); + bool _setp_391 = _setp_391_bits != 0; + float nm_err_2 = ((_setp_391) ? nm_es_2 : nm_ex_2); + float _add_394; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_394; s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4372,8 +4338,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_363 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_363), "f"(off_acc[0]) : "memory"); + uint32_t _addr_395 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_395), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4399,324 +4365,137 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1292 = r_297[0]; - float gc_1293 = gn_1292 - 21.640425f; - int eb_1294 = 81920 + th * -32768; - int ew_1295 = eb_1294 >> 2; - int qbase_1296 = pool_f32_addr + (unsigned int)(eb_1294 + 12288 + dh_lane * 1024); - int sb8_1297 = 16 + 8 * th; - int gb8_1298 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1297 * 128); - int kb8_1299 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1297 * 128); - float dbp_1300[8]; - float _tmem_load_20[8]; - tmem_ld_x8(&_tmem_load_20[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); - float _tmem_load_21[8]; - tmem_ld_x8(&_tmem_load_21[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); - float _tmem_load_22[8]; - tmem_ld_x8(&_tmem_load_22[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); - float _tmem_load_23[8]; - tmem_ld_x8(&_tmem_load_23[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); - float _tmem_load_24[8]; - tmem_ld_x8(&_tmem_load_24[0], taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th)); - float btv_1301[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1301[0])), "=r"(*reinterpret_cast(&btv_1301[(0) + 1])), "=r"(*reinterpret_cast(&btv_1301[(0) + 2])), "=r"(*reinterpret_cast(&btv_1301[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1301[4])), "=r"(*reinterpret_cast(&btv_1301[(4) + 1])), "=r"(*reinterpret_cast(&btv_1301[(4) + 2])), "=r"(*reinterpret_cast(&btv_1301[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th) + 16))); + float r_276[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1 = r_276[0]; + float gc_1 = gn_1 - 21.640425f; + int eb_1 = 81920 + th * -32768; + int ew_1 = eb_1 >> 2; + int qbase_1 = pool_f32_addr + (unsigned int)(eb_1 + 12288 + dh_lane * 1024); + int sb8_1 = 16 + 8 * th; + int gb8_1 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1 * 128); + int kb8_1 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1 * 128); + int c_qa_0 = 64 + 8 * th; + int c_ka_0 = 80 + 8 * th; + int c_qd_0 = 96 + 8 * th; + int c_kd_0 = 112 + 8 * th; + int c_dk_0 = 16 + 8 * th; + int c_bp_0 = c_ka_0; + float s_run_1[1]; + s_run_1[0] = 0.0f; + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_1 = 7 - k_1; + float a_qd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1))); + float a_kd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_0 + j_1))); + float a_dk_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_0 + j_1))); + float a_qa_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa_0 + j_1))); + float a_ka_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka_0 + j_1))); + float btr_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_1 + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_1 + j_1; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1 = __uint_as_float(r_279[0] << 16); + float q_m_1 = ((t_1 >= clen) ? 0.0f : q_t_1); + float dqf_1 = pool_f32[ew_1 + 128 * j_1 + d_lane]; + float dkf_1 = pool_f32[ew_1 + (1024 + 128 * j_1) + d_lane]; + float dgf_1 = pool_f32[ew_1 + (2048 + 128 * j_1) + d_lane]; + float bt_t_1 = btr_1[0]; + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float e_off_1 = approx_exp2(g_t_1 - gn_1); + float dq2_1 = __fmaf_rn(a_qa_1[0], e_off_1, a_qd_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(a_ka_1[0], e_off_1, a_kd_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + a_dk_1[0]; + float qe_8 = q_m_1 * e_dg_1; + float kp_20 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_1; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, a_qd_1[0], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_1) * a_kd_1[0]); + float dg_o_1 = __fmaf_rn(q_m_1, dq2_1, __fmaf_rn(dk2b_1 - a_dk_1[0], k_t_1, cr_8)) + dgf_1; + s_run_1[0] = s_run_1[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1)), "f"(s_run_1[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_1)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(81920 - 32768 * th >> 2) + (128 * j_1 + d_lane)] = dq_o_1; + pool_f32[(81920 - 32768 * th >> 2) + (1024 + 128 * j_1 + d_lane)] = dk_o_1; + } + float tot_1 = s_run_1[0]; + float dbp_1[8]; + tmem_ld_x8(&dbp_1[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1302[1]; - s_run_1302[0] = 0.0f; - int t_8 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_1295 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_1295 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_1295 + 2944 + d_lane]; - float bt_t_8 = btv_1301[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_1292); - float e_dg_8 = approx_exp2(g_t_8 - gc_1293); - float dq2_8 = __fmaf_rn(_tmem_load_20[7], e_off_8, _tmem_load_22[7] * e_dg_8); - float dk2_8 = __fmaf_rn(_tmem_load_21[7], e_off_8, _tmem_load_23[7] * e_dg_8); - dbp_1300[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_24[7]; - float dg_o_8 = __fmaf_rn(q_t_1_16, dq2_8, (dk2b_8 - _tmem_load_24[7]) * k_t_8) + dgf_8; - s_run_1302[0] = s_run_1302[0] + dg_o_8; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_8; - pool_f32[(81920 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_8; - int t_9 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_1295 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_1295 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_1295 + 2816 + d_lane]; - float bt_t_9 = btv_1301[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_1292); - float e_dg_9 = approx_exp2(g_t_9 - gc_1293); - float dq2_9 = __fmaf_rn(_tmem_load_20[6], e_off_9, _tmem_load_22[6] * e_dg_9); - float dk2_9 = __fmaf_rn(_tmem_load_21[6], e_off_9, _tmem_load_23[6] * e_dg_9); - dbp_1300[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[6]; - float dg_o_9 = __fmaf_rn(q_t_1_17, dq2_9, (dk2b_9 - _tmem_load_24[6]) * k_t_9) + dgf_9; - s_run_1302[0] = s_run_1302[0] + dg_o_9; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_9; - pool_f32[(81920 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_9; - int t_10 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_1295 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_1295 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_1295 + 2688 + d_lane]; - float bt_t_10 = btv_1301[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_1292); - float e_dg_10 = approx_exp2(g_t_10 - gc_1293); - float dq2_10 = __fmaf_rn(_tmem_load_20[5], e_off_10, _tmem_load_22[5] * e_dg_10); - float dk2_10 = __fmaf_rn(_tmem_load_21[5], e_off_10, _tmem_load_23[5] * e_dg_10); - dbp_1300[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[5]; - float dg_o_10 = __fmaf_rn(q_t_1_18, dq2_10, (dk2b_10 - _tmem_load_24[5]) * k_t_10) + dgf_10; - s_run_1302[0] = s_run_1302[0] + dg_o_10; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_10; - pool_f32[(81920 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_10; - int t_11 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_1295 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_1295 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_1295 + 2560 + d_lane]; - float bt_t_11 = btv_1301[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_1292); - float e_dg_11 = approx_exp2(g_t_11 - gc_1293); - float dq2_11 = __fmaf_rn(_tmem_load_20[4], e_off_11, _tmem_load_22[4] * e_dg_11); - float dk2_11 = __fmaf_rn(_tmem_load_21[4], e_off_11, _tmem_load_23[4] * e_dg_11); - dbp_1300[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[4]; - float dg_o_11 = __fmaf_rn(q_t_1_19, dq2_11, (dk2b_11 - _tmem_load_24[4]) * k_t_11) + dgf_11; - s_run_1302[0] = s_run_1302[0] + dg_o_11; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_11; - pool_f32[(81920 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_11; - int t_12 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_1295 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_1295 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_1295 + 2432 + d_lane]; - float bt_t_12 = btv_1301[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_1292); - float e_dg_12 = approx_exp2(g_t_12 - gc_1293); - float dq2_12 = __fmaf_rn(_tmem_load_20[3], e_off_12, _tmem_load_22[3] * e_dg_12); - float dk2_12 = __fmaf_rn(_tmem_load_21[3], e_off_12, _tmem_load_23[3] * e_dg_12); - dbp_1300[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[3]; - float dg_o_12 = __fmaf_rn(q_t_1_20, dq2_12, (dk2b_12 - _tmem_load_24[3]) * k_t_12) + dgf_12; - s_run_1302[0] = s_run_1302[0] + dg_o_12; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_12; - pool_f32[(81920 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_12; - int t_13 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_1295 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_1295 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_1295 + 2304 + d_lane]; - float bt_t_13 = btv_1301[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_1292); - float e_dg_13 = approx_exp2(g_t_13 - gc_1293); - float dq2_13 = __fmaf_rn(_tmem_load_20[2], e_off_13, _tmem_load_22[2] * e_dg_13); - float dk2_13 = __fmaf_rn(_tmem_load_21[2], e_off_13, _tmem_load_23[2] * e_dg_13); - dbp_1300[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[2]; - float dg_o_13 = __fmaf_rn(q_t_1_21, dq2_13, (dk2b_13 - _tmem_load_24[2]) * k_t_13) + dgf_13; - s_run_1302[0] = s_run_1302[0] + dg_o_13; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_13; - pool_f32[(81920 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_13; - int t_14 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_1295 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_1295 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_1295 + 2176 + d_lane]; - float bt_t_14 = btv_1301[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_1292); - float e_dg_14 = approx_exp2(g_t_14 - gc_1293); - float dq2_14 = __fmaf_rn(_tmem_load_20[1], e_off_14, _tmem_load_22[1] * e_dg_14); - float dk2_14 = __fmaf_rn(_tmem_load_21[1], e_off_14, _tmem_load_23[1] * e_dg_14); - dbp_1300[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[1]; - float dg_o_14 = __fmaf_rn(q_t_1_22, dq2_14, (dk2b_14 - _tmem_load_24[1]) * k_t_14) + dgf_14; - s_run_1302[0] = s_run_1302[0] + dg_o_14; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_14; - pool_f32[(81920 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_14; - int t_15 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_1295 + d_lane]; - float dkf_15 = pool_f32[ew_1295 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_1295 + 2048 + d_lane]; - float bt_t_15 = btv_1301[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_1292); - float e_dg_15 = approx_exp2(g_t_15 - gc_1293); - float dq2_15 = __fmaf_rn(_tmem_load_20[0], e_off_15, _tmem_load_22[0] * e_dg_15); - float dk2_15 = __fmaf_rn(_tmem_load_21[0], e_off_15, _tmem_load_23[0] * e_dg_15); - dbp_1300[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[0]; - float dg_o_15 = __fmaf_rn(q_t_1_23, dq2_15, (dk2b_15 - _tmem_load_24[0]) * k_t_15) + dgf_15; - s_run_1302[0] = s_run_1302[0] + dg_o_15; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1302[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(81920 - 32768 * th >> 2) + d_lane] = dq_o_15; - pool_f32[(81920 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_15; - float tot_1471 = s_run_1302[0]; - int b4_1472 = lane_1 >> 4 & 1; - int b3_1473 = lane_1 >> 3 & 1; - int b2_1474 = lane_1 >> 2 & 1; - float v4_1475[4]; - float send_6 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_364 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_364; - float send_7 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_365 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_365; - float send_8 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_366 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_366; - float send_9 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_367; - float v2_1480[2]; - float send_10 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_368; - float send_11 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_369; - float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_370; - float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_371; - float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_372; + int b4_1 = lane_1 >> 4 & 1; + int b3_1 = lane_1 >> 3 & 1; + int b2_1 = lane_1 >> 2 & 1; + float v4_1[4]; + float send_4 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); + float _shfl_xor_sync_396 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_396; + float send_5 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); + float _shfl_xor_sync_397 = __shfl_xor_sync(0xFFFFFFFF, send_5, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_397; + float send_6 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); + float _shfl_xor_sync_398 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_398; + float send_7 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); + float _shfl_xor_sync_399 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_399; + float v2_16[2]; + float send2_2 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); + float _shfl_xor_sync_400 = __shfl_xor_sync(0xFFFFFFFF, send2_2, 8); + v2_16[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_400; + float send2_3 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); + float _shfl_xor_sync_401 = __shfl_xor_sync(0xFFFFFFFF, send2_3, 8); + v2_16[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_401; + float send1_0 = ((b2_1 == 0) ? v2_16[1] : v2_16[0]); + float _shfl_xor_sync_402 = __shfl_xor_sync(0xFFFFFFFF, send1_0, 4); + float v1_1 = ((b2_1 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_402; + float _shfl_xor_sync_403 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_403; + float _shfl_xor_sync_404 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_404; asm volatile("tcgen05.fence::before_thread_sync;"); if (16 + 8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + (16 + 8 * th)) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(81920 - 32768 * th >> 2) + d_lane]; @@ -4771,8 +4550,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_373 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_373), "f"(tot_1471) : "memory"); + uint32_t _addr_405 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_405), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4790,8 +4569,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other_0[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other_0[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi_1 = ((th == 1) ? tot_1471 : t_other_0[0]); - float t_lo_1 = ((th == 1) ? t_other_0[0] : tot_1471); + float t_hi_1 = ((th == 1) ? tot_1 : t_other_0[0]); + float t_lo_1 = ((th == 1) ? t_other_0[0] : tot_1); float offset_1 = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi_1); float sfx_1[8]; tmem_ld_x8(&sfx_1[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); @@ -4804,132 +4583,132 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_374 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_406 = make_float2(gacc[4], gacc[4]); float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_374), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_377; - asm("abs.f32 %0, %1;" : "=f"(_abs_377) : "f"(rp_z_7.x)); - float _neg_376; - asm("neg.f32 %0, %1;" : "=f"(_neg_376) : "f"(_abs_377)); - float _exp_375 = __nv_expf(_neg_376); - float _abs_380; - asm("abs.f32 %0, %1;" : "=f"(_abs_380) : "f"(rp_z_7.y)); - float _neg_379; - asm("neg.f32 %0, %1;" : "=f"(_neg_379) : "f"(_abs_380)); - float _exp_378 = __nv_expf(_neg_379); - float2 rp_e_7 = make_float2(_exp_375, _exp_378); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_406), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_409; + asm("abs.f32 %0, %1;" : "=f"(_abs_409) : "f"(rp_z_7.x)); + float _neg_408; + asm("neg.f32 %0, %1;" : "=f"(_neg_408) : "f"(_abs_409)); + float _exp_407 = __nv_expf(_neg_408); + float _abs_412; + asm("abs.f32 %0, %1;" : "=f"(_abs_412) : "f"(rp_z_7.y)); + float _neg_411; + asm("neg.f32 %0, %1;" : "=f"(_neg_411) : "f"(_abs_412)); + float _exp_410 = __nv_expf(_neg_411); + float2 rp_e_7 = make_float2(_exp_407, _exp_410); float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_381 = approx_rcp(rp_den_7.x); - float _rcp_382 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_383 = make_float2(_rcp_381, _rcp_382); + float _rcp_413 = approx_rcp(rp_den_7.x); + float _rcp_414 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_415 = make_float2(_rcp_413, _rcp_414); float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_383)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_415)); float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_384 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_416 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_384)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_416)); u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_385 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_417 = make_float2(gacc[4], gacc[4]); float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_385), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_388; - asm("abs.f32 %0, %1;" : "=f"(_abs_388) : "f"(rp_z_8.x)); - float _neg_387; - asm("neg.f32 %0, %1;" : "=f"(_neg_387) : "f"(_abs_388)); - float _exp_386 = __nv_expf(_neg_387); - float _abs_391; - asm("abs.f32 %0, %1;" : "=f"(_abs_391) : "f"(rp_z_8.y)); - float _neg_390; - asm("neg.f32 %0, %1;" : "=f"(_neg_390) : "f"(_abs_391)); - float _exp_389 = __nv_expf(_neg_390); - float2 rp_e_8 = make_float2(_exp_386, _exp_389); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_417), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_420; + asm("abs.f32 %0, %1;" : "=f"(_abs_420) : "f"(rp_z_8.x)); + float _neg_419; + asm("neg.f32 %0, %1;" : "=f"(_neg_419) : "f"(_abs_420)); + float _exp_418 = __nv_expf(_neg_419); + float _abs_423; + asm("abs.f32 %0, %1;" : "=f"(_abs_423) : "f"(rp_z_8.y)); + float _neg_422; + asm("neg.f32 %0, %1;" : "=f"(_neg_422) : "f"(_abs_423)); + float _exp_421 = __nv_expf(_neg_422); + float2 rp_e_8 = make_float2(_exp_418, _exp_421); float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_392 = approx_rcp(rp_den_8.x); - float _rcp_393 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_394 = make_float2(_rcp_392, _rcp_393); + float _rcp_424 = approx_rcp(rp_den_8.x); + float _rcp_425 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_426 = make_float2(_rcp_424, _rcp_425); float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_394)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_426)); float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_395 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_427 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_395)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_427)); u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_396 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_428 = make_float2(gacc[4], gacc[4]); float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_396), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_399; - asm("abs.f32 %0, %1;" : "=f"(_abs_399) : "f"(rp_z_9.x)); - float _neg_398; - asm("neg.f32 %0, %1;" : "=f"(_neg_398) : "f"(_abs_399)); - float _exp_397 = __nv_expf(_neg_398); - float _abs_402; - asm("abs.f32 %0, %1;" : "=f"(_abs_402) : "f"(rp_z_9.y)); - float _neg_401; - asm("neg.f32 %0, %1;" : "=f"(_neg_401) : "f"(_abs_402)); - float _exp_400 = __nv_expf(_neg_401); - float2 rp_e_9 = make_float2(_exp_397, _exp_400); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_428), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_431; + asm("abs.f32 %0, %1;" : "=f"(_abs_431) : "f"(rp_z_9.x)); + float _neg_430; + asm("neg.f32 %0, %1;" : "=f"(_neg_430) : "f"(_abs_431)); + float _exp_429 = __nv_expf(_neg_430); + float _abs_434; + asm("abs.f32 %0, %1;" : "=f"(_abs_434) : "f"(rp_z_9.y)); + float _neg_433; + asm("neg.f32 %0, %1;" : "=f"(_neg_433) : "f"(_abs_434)); + float _exp_432 = __nv_expf(_neg_433); + float2 rp_e_9 = make_float2(_exp_429, _exp_432); float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_403 = approx_rcp(rp_den_9.x); - float _rcp_404 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_405 = make_float2(_rcp_403, _rcp_404); + float _rcp_435 = approx_rcp(rp_den_9.x); + float _rcp_436 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_437 = make_float2(_rcp_435, _rcp_436); float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_405)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_437)); float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_406 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_438 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_406)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_438)); u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_407 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_439 = make_float2(gacc[4], gacc[4]); float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_407), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_410; - asm("abs.f32 %0, %1;" : "=f"(_abs_410) : "f"(rp_z_10.x)); - float _neg_409; - asm("neg.f32 %0, %1;" : "=f"(_neg_409) : "f"(_abs_410)); - float _exp_408 = __nv_expf(_neg_409); - float _abs_413; - asm("abs.f32 %0, %1;" : "=f"(_abs_413) : "f"(rp_z_10.y)); - float _neg_412; - asm("neg.f32 %0, %1;" : "=f"(_neg_412) : "f"(_abs_413)); - float _exp_411 = __nv_expf(_neg_412); - float2 rp_e_10 = make_float2(_exp_408, _exp_411); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_439), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_442; + asm("abs.f32 %0, %1;" : "=f"(_abs_442) : "f"(rp_z_10.x)); + float _neg_441; + asm("neg.f32 %0, %1;" : "=f"(_neg_441) : "f"(_abs_442)); + float _exp_440 = __nv_expf(_neg_441); + float _abs_445; + asm("abs.f32 %0, %1;" : "=f"(_abs_445) : "f"(rp_z_10.y)); + float _neg_444; + asm("neg.f32 %0, %1;" : "=f"(_neg_444) : "f"(_abs_445)); + float _exp_443 = __nv_expf(_neg_444); + float2 rp_e_10 = make_float2(_exp_440, _exp_443); float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_414 = approx_rcp(rp_den_10.x); - float _rcp_415 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_416 = make_float2(_rcp_414, _rcp_415); + float _rcp_446 = approx_rcp(rp_den_10.x); + float _rcp_447 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_448 = make_float2(_rcp_446, _rcp_447); float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_416)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_448)); float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_417 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_449 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_417)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_449)); u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -4944,54 +4723,54 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_418; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_418) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_418; + uint32_t _prmt_450; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_450) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_450; int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_419 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_419)); - *(reinterpret_cast(_gst_419) + (0)) = pq_8[0]; + auto _gst_451 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_451)); + *(reinterpret_cast(_gst_451) + (0)) = pq_8[0]; } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_420; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_420) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_420; + uint32_t _prmt_452; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_452) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_452; int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_421 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_421)); - *(reinterpret_cast(_gst_421) + (0)) = pq_9[0]; + auto _gst_453 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_453)); + *(reinterpret_cast(_gst_453) + (0)) = pq_9[0]; } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_422; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_422) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_422; + uint32_t _prmt_454; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_454) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_454; int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_423 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_423)); - *(reinterpret_cast(_gst_423) + (0)) = pq_10[0]; + auto _gst_455 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_455)); + *(reinterpret_cast(_gst_455) + (0)) = pq_10[0]; } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_424; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_424) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_424; + uint32_t _prmt_456; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_456) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_456; int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_425 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_425)); - *(reinterpret_cast(_gst_425) + (0)) = pq_11[0]; + auto _gst_457 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_457)); + *(reinterpret_cast(_gst_457) + (0)) = pq_11[0]; } } int t_a_16 = 16 + 8 * th; @@ -5056,47 +4835,47 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_426; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_426) : "f"(s_a_1), "f"(nm_t_3)); + float _sub_458; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_458) : "f"(s_a_1), "f"(nm_t_3)); float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_426), "f"(pa_1)); - float _sub_427; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_427) : "f"(pa_1), "f"(nm_t_3)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_458), "f"(pa_1)); + float _sub_459; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_459) : "f"(pa_1), "f"(nm_t_3)); float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_427), "f"(s_a_1)); - float _abs_429; - asm("abs.f32 %0, %1;" : "=f"(_abs_429) : "f"(s_a_1)); - float _abs_430; - asm("abs.f32 %0, %1;" : "=f"(_abs_430) : "f"(pa_1)); - uint32_t _setp_428_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_428_bits) : "f"(_abs_429), "f"(_abs_430)); - bool _setp_428 = _setp_428_bits != 0; - float nm_err_3 = ((_setp_428) ? nm_es_3 : nm_ex_3); - float _add_431; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_431) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_431; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_459), "f"(s_a_1)); + float _abs_461; + asm("abs.f32 %0, %1;" : "=f"(_abs_461) : "f"(s_a_1)); + float _abs_462; + asm("abs.f32 %0, %1;" : "=f"(_abs_462) : "f"(pa_1)); + uint32_t _setp_460_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_460_bits) : "f"(_abs_461), "f"(_abs_462)); + bool _setp_460 = _setp_460_bits != 0; + float nm_err_3 = ((_setp_460) ? nm_es_3 : nm_ex_3); + float _add_463; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_463) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_463; s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_432; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_432) : "f"(s_b_1), "f"(nm_t_4)); + float _sub_464; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_464) : "f"(s_b_1), "f"(nm_t_4)); float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_432), "f"(pb_1)); - float _sub_433; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_433) : "f"(pb_1), "f"(nm_t_4)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_464), "f"(pb_1)); + float _sub_465; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_465) : "f"(pb_1), "f"(nm_t_4)); float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_433), "f"(s_b_1)); - float _abs_435; - asm("abs.f32 %0, %1;" : "=f"(_abs_435) : "f"(s_b_1)); - float _abs_436; - asm("abs.f32 %0, %1;" : "=f"(_abs_436) : "f"(pb_1)); - uint32_t _setp_434_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_434_bits) : "f"(_abs_435), "f"(_abs_436)); - bool _setp_434 = _setp_434_bits != 0; - float nm_err_4 = ((_setp_434) ? nm_es_4 : nm_ex_4); - float _add_437; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_437) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_437; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_465), "f"(s_b_1)); + float _abs_467; + asm("abs.f32 %0, %1;" : "=f"(_abs_467) : "f"(s_b_1)); + float _abs_468; + asm("abs.f32 %0, %1;" : "=f"(_abs_468) : "f"(pb_1)); + uint32_t _setp_466_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_466_bits) : "f"(_abs_467), "f"(_abs_468)); + bool _setp_466 = _setp_466_bits != 0; + float nm_err_4 = ((_setp_466) ? nm_es_4 : nm_ex_4); + float _add_469; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_469) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_469; s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5104,8 +4883,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_438 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_438), "f"(off_acc[0]) : "memory"); + uint32_t _addr_470 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_470), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5129,320 +4908,124 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1514 = r_322[0]; - float gc_1515 = gn_1514 - 21.640425f; - int eb_1516 = 98304 + th * -32768; - int ew_1517 = eb_1516 >> 2; - int qbase_1518 = pool_f32_addr + (unsigned int)(eb_1516 + 12288 + dh_lane * 1024); - int sb8_1519 = 8 * th; - int gb8_1520 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1519 * 128); - int kb8_1521 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1519 * 128); - float dbp_1522[8]; - float _tmem_load_26[8]; - tmem_ld_x8(&_tmem_load_26[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float _tmem_load_27[8]; - tmem_ld_x8(&_tmem_load_27[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float _tmem_load_28[8]; - tmem_ld_x8(&_tmem_load_28[0], taddr + (unsigned int)row_base + (unsigned int)(8 * th)); - float btv_1523[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1523[0])), "=r"(*reinterpret_cast(&btv_1523[(0) + 1])), "=r"(*reinterpret_cast(&btv_1523[(0) + 2])), "=r"(*reinterpret_cast(&btv_1523[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1523[4])), "=r"(*reinterpret_cast(&btv_1523[(4) + 1])), "=r"(*reinterpret_cast(&btv_1523[(4) + 2])), "=r"(*reinterpret_cast(&btv_1523[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th) + 16))); + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_2 = r_280[0]; + float gc_2 = gn_2 - 21.640425f; + int eb_2 = 98304 + th * -32768; + int ew_2 = eb_2 >> 2; + int qbase_2 = pool_f32_addr + (unsigned int)(eb_2 + 12288 + dh_lane * 1024); + int sb8_2 = 8 * th; + int gb8_2 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_2 * 128); + int kb8_2 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_2 * 128); + int c_qa_1 = 8 * th; + int c_ka_1 = 8 * th; + int c_qd_1 = 128 + 8 * th; + int c_kd_1 = 144 + 8 * th; + int c_dk_1 = 8 * th; + int c_bp_1 = c_kd_1; + float s_run_2[1]; + s_run_2[0] = 0.0f; + #pragma unroll 1 + for (int k_2 = 0; k_2 < 8; k_2++) { + int j_2 = 7 - k_2; + float a_qd_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_1 + j_2))); + float a_kd_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_1 + j_2))); + float a_dk_2[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_2[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_1 + j_2))); + float btr_2[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_2[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_2 + j_2)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_2 = sb8_2 + j_2; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 * j_2 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_2 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 * j_2 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_2 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 * j_2 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_2 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_283[0] << 16); + float q_m_2 = ((t_2 >= clen) ? 0.0f : q_t_2); + float dqf_2 = pool_f32[ew_2 + 128 * j_2 + d_lane]; + float dkf_2 = pool_f32[ew_2 + (1024 + 128 * j_2) + d_lane]; + float dgf_2 = pool_f32[ew_2 + (2048 + 128 * j_2) + d_lane]; + float bt_t_2 = btr_2[0]; + float e_dg_2 = approx_exp2(g_t_2 - gc_2); + float dq2_2 = a_qd_2[0] * e_dg_2; + float dk2_2 = a_kd_2[0] * e_dg_2; + float dbv_2[1]; + dbv_2[0] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + a_dk_2[0]; + float qe_9 = q_m_2 * e_dg_2; + float kp_21 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, a_qd_2[0], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_2) * a_kd_2[0]); + float dg_o_2 = __fmaf_rn(q_m_2, dq2_2, __fmaf_rn(dk2b_2 - a_dk_2[0], k_t_2, cr_9)) + dgf_2; + s_run_2[0] = s_run_2[0] + dg_o_2; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_1 + j_2)), "f"(s_run_2[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_1 + j_2)), "f"(dbv_2[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + pool_f32[(98304 - 32768 * th >> 2) + (128 * j_2 + d_lane)] = dq_o_2; + pool_f32[(98304 - 32768 * th >> 2) + (1024 + 128 * j_2 + d_lane)] = dk_o_2; + } + float tot_2 = s_run_2[0]; + float dbp_2[8]; + tmem_ld_x8(&dbp_2[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_1); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1524[1]; - s_run_1524[0] = 0.0f; - int t_16 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1517 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1517 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1517 + 2944 + d_lane]; - float bt_t_16 = btv_1523[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1514); - float e_dg_16 = approx_exp2(g_t_16 - gc_1515); - float dq2_16 = _tmem_load_26[7] * e_dg_16; - float dk2_16 = _tmem_load_27[7] * e_dg_16; - dbp_1522[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_28[7]; - float dg_o_16 = __fmaf_rn(q_t_1_24, dq2_16, (dk2b_16 - _tmem_load_28[7]) * k_t_16) + dgf_16; - s_run_1524[0] = s_run_1524[0] + dg_o_16; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (896 + d_lane)] = dq_o_16; - pool_f32[(98304 - 32768 * th >> 2) + (1920 + d_lane)] = dk_o_16; - int t_17 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1517 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1517 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1517 + 2816 + d_lane]; - float bt_t_17 = btv_1523[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1514); - float e_dg_17 = approx_exp2(g_t_17 - gc_1515); - float dq2_17 = _tmem_load_26[6] * e_dg_17; - float dk2_17 = _tmem_load_27[6] * e_dg_17; - dbp_1522[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[6]; - float dg_o_17 = __fmaf_rn(q_t_1_25, dq2_17, (dk2b_17 - _tmem_load_28[6]) * k_t_17) + dgf_17; - s_run_1524[0] = s_run_1524[0] + dg_o_17; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (768 + d_lane)] = dq_o_17; - pool_f32[(98304 - 32768 * th >> 2) + (1792 + d_lane)] = dk_o_17; - int t_18 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1517 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1517 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1517 + 2688 + d_lane]; - float bt_t_18 = btv_1523[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1514); - float e_dg_18 = approx_exp2(g_t_18 - gc_1515); - float dq2_18 = _tmem_load_26[5] * e_dg_18; - float dk2_18 = _tmem_load_27[5] * e_dg_18; - dbp_1522[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[5]; - float dg_o_18 = __fmaf_rn(q_t_1_26, dq2_18, (dk2b_18 - _tmem_load_28[5]) * k_t_18) + dgf_18; - s_run_1524[0] = s_run_1524[0] + dg_o_18; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (640 + d_lane)] = dq_o_18; - pool_f32[(98304 - 32768 * th >> 2) + (1664 + d_lane)] = dk_o_18; - int t_19 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1517 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1517 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1517 + 2560 + d_lane]; - float bt_t_19 = btv_1523[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1514); - float e_dg_19 = approx_exp2(g_t_19 - gc_1515); - float dq2_19 = _tmem_load_26[4] * e_dg_19; - float dk2_19 = _tmem_load_27[4] * e_dg_19; - dbp_1522[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[4]; - float dg_o_19 = __fmaf_rn(q_t_1_27, dq2_19, (dk2b_19 - _tmem_load_28[4]) * k_t_19) + dgf_19; - s_run_1524[0] = s_run_1524[0] + dg_o_19; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (512 + d_lane)] = dq_o_19; - pool_f32[(98304 - 32768 * th >> 2) + (1536 + d_lane)] = dk_o_19; - int t_20 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1517 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1517 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1517 + 2432 + d_lane]; - float bt_t_20 = btv_1523[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1514); - float e_dg_20 = approx_exp2(g_t_20 - gc_1515); - float dq2_20 = _tmem_load_26[3] * e_dg_20; - float dk2_20 = _tmem_load_27[3] * e_dg_20; - dbp_1522[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[3]; - float dg_o_20 = __fmaf_rn(q_t_1_28, dq2_20, (dk2b_20 - _tmem_load_28[3]) * k_t_20) + dgf_20; - s_run_1524[0] = s_run_1524[0] + dg_o_20; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (384 + d_lane)] = dq_o_20; - pool_f32[(98304 - 32768 * th >> 2) + (1408 + d_lane)] = dk_o_20; - int t_21 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1517 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1517 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1517 + 2304 + d_lane]; - float bt_t_21 = btv_1523[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1514); - float e_dg_21 = approx_exp2(g_t_21 - gc_1515); - float dq2_21 = _tmem_load_26[2] * e_dg_21; - float dk2_21 = _tmem_load_27[2] * e_dg_21; - dbp_1522[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[2]; - float dg_o_21 = __fmaf_rn(q_t_1_29, dq2_21, (dk2b_21 - _tmem_load_28[2]) * k_t_21) + dgf_21; - s_run_1524[0] = s_run_1524[0] + dg_o_21; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (256 + d_lane)] = dq_o_21; - pool_f32[(98304 - 32768 * th >> 2) + (1280 + d_lane)] = dk_o_21; - int t_22 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1517 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1517 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1517 + 2176 + d_lane]; - float bt_t_22 = btv_1523[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1514); - float e_dg_22 = approx_exp2(g_t_22 - gc_1515); - float dq2_22 = _tmem_load_26[1] * e_dg_22; - float dk2_22 = _tmem_load_27[1] * e_dg_22; - dbp_1522[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[1]; - float dg_o_22 = __fmaf_rn(q_t_1_30, dq2_22, (dk2b_22 - _tmem_load_28[1]) * k_t_22) + dgf_22; - s_run_1524[0] = s_run_1524[0] + dg_o_22; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + (128 + d_lane)] = dq_o_22; - pool_f32[(98304 - 32768 * th >> 2) + (1152 + d_lane)] = dk_o_22; - int t_23 = 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1517 + d_lane]; - float dkf_23 = pool_f32[ew_1517 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1517 + 2048 + d_lane]; - float bt_t_23 = btv_1523[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1514); - float e_dg_23 = approx_exp2(g_t_23 - gc_1515); - float dq2_23 = _tmem_load_26[0] * e_dg_23; - float dk2_23 = _tmem_load_27[0] * e_dg_23; - dbp_1522[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[0]; - float dg_o_23 = __fmaf_rn(q_t_1_31, dq2_23, (dk2b_23 - _tmem_load_28[0]) * k_t_23) + dgf_23; - s_run_1524[0] = s_run_1524[0] + dg_o_23; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_1524[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - pool_f32[(98304 - 32768 * th >> 2) + d_lane] = dq_o_23; - pool_f32[(98304 - 32768 * th >> 2) + (1024 + d_lane)] = dk_o_23; - float tot_1693 = s_run_1524[0]; - int b4_1694 = lane_1 >> 4 & 1; - int b3_1695 = lane_1 >> 3 & 1; - int b2_1696 = lane_1 >> 2 & 1; - float v4_1697[4]; - float send_12 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_439 = __shfl_xor_sync(0xFFFFFFFF, send_12, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_439; - float send_13 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_440 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_440; - float send_14 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_441 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_441; - float send_15 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_442 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_442; - float v2_1702[2]; - float send_16 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_443 = __shfl_xor_sync(0xFFFFFFFF, send_16, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_443; - float send_17 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_444 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_444; - float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_445 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_445; - float _shfl_xor_sync_446 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_446; - float _shfl_xor_sync_447 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_447; + int b4_2 = lane_1 >> 4 & 1; + int b3_2 = lane_1 >> 3 & 1; + int b2_2 = lane_1 >> 2 & 1; + float v4_2[4]; + float send_8 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); + float _shfl_xor_sync_471 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_471; + float send_9 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); + float _shfl_xor_sync_472 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_472; + float send_10 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); + float _shfl_xor_sync_473 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_473; + float send_11 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); + float _shfl_xor_sync_474 = __shfl_xor_sync(0xFFFFFFFF, send_11, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_474; + float v2_17[2]; + float send2_4 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); + float _shfl_xor_sync_475 = __shfl_xor_sync(0xFFFFFFFF, send2_4, 8); + v2_17[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_475; + float send2_5 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); + float _shfl_xor_sync_476 = __shfl_xor_sync(0xFFFFFFFF, send2_5, 8); + v2_17[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_476; + float send1_1 = ((b2_2 == 0) ? v2_17[1] : v2_17[0]); + float _shfl_xor_sync_477 = __shfl_xor_sync(0xFFFFFFFF, send1_1, 4); + float v1_2 = ((b2_2 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_477; + float _shfl_xor_sync_478 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_478; + float _shfl_xor_sync_479 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_479; if (8 * th < clen) { *(reinterpret_cast(dq_f + (((long long)(row0 + 8 * th) * (long long)num_heads + (long long)hv) * 128 + (long long)d_lane)) + (0)) = pool_f32[(98304 - 32768 * th >> 2) + d_lane]; } @@ -5495,8 +5078,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_448 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_448), "f"(tot_1693) : "memory"); + uint32_t _addr_480 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_480), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5514,8 +5097,8 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other_1[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other_1[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi_2 = ((th == 1) ? tot_1693 : t_other_1[0]); - float t_lo_2 = ((th == 1) ? t_other_1[0] : tot_1693); + float t_hi_2 = ((th == 1) ? tot_2 : t_other_1[0]); + float t_lo_2 = ((th == 1) ? t_other_1[0] : tot_2); float offset_2 = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi_2); float sfx_2[8]; tmem_ld_x8(&sfx_2[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); @@ -5528,132 +5111,132 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_449 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_481 = make_float2(gacc[4], gacc[4]); float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_449), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_452; - asm("abs.f32 %0, %1;" : "=f"(_abs_452) : "f"(rp_z_11.x)); - float _neg_451; - asm("neg.f32 %0, %1;" : "=f"(_neg_451) : "f"(_abs_452)); - float _exp_450 = __nv_expf(_neg_451); - float _abs_455; - asm("abs.f32 %0, %1;" : "=f"(_abs_455) : "f"(rp_z_11.y)); - float _neg_454; - asm("neg.f32 %0, %1;" : "=f"(_neg_454) : "f"(_abs_455)); - float _exp_453 = __nv_expf(_neg_454); - float2 rp_e_11 = make_float2(_exp_450, _exp_453); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_481), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_484; + asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(rp_z_11.x)); + float _neg_483; + asm("neg.f32 %0, %1;" : "=f"(_neg_483) : "f"(_abs_484)); + float _exp_482 = __nv_expf(_neg_483); + float _abs_487; + asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(rp_z_11.y)); + float _neg_486; + asm("neg.f32 %0, %1;" : "=f"(_neg_486) : "f"(_abs_487)); + float _exp_485 = __nv_expf(_neg_486); + float2 rp_e_11 = make_float2(_exp_482, _exp_485); float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_456 = approx_rcp(rp_den_11.x); - float _rcp_457 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_458 = make_float2(_rcp_456, _rcp_457); + float _rcp_488 = approx_rcp(rp_den_11.x); + float _rcp_489 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_490 = make_float2(_rcp_488, _rcp_489); float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_490)); float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_459 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_491 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_459)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_491)); u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_460 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_492 = make_float2(gacc[4], gacc[4]); float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_460), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_463; - asm("abs.f32 %0, %1;" : "=f"(_abs_463) : "f"(rp_z_12.x)); - float _neg_462; - asm("neg.f32 %0, %1;" : "=f"(_neg_462) : "f"(_abs_463)); - float _exp_461 = __nv_expf(_neg_462); - float _abs_466; - asm("abs.f32 %0, %1;" : "=f"(_abs_466) : "f"(rp_z_12.y)); - float _neg_465; - asm("neg.f32 %0, %1;" : "=f"(_neg_465) : "f"(_abs_466)); - float _exp_464 = __nv_expf(_neg_465); - float2 rp_e_12 = make_float2(_exp_461, _exp_464); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_492), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_495; + asm("abs.f32 %0, %1;" : "=f"(_abs_495) : "f"(rp_z_12.x)); + float _neg_494; + asm("neg.f32 %0, %1;" : "=f"(_neg_494) : "f"(_abs_495)); + float _exp_493 = __nv_expf(_neg_494); + float _abs_498; + asm("abs.f32 %0, %1;" : "=f"(_abs_498) : "f"(rp_z_12.y)); + float _neg_497; + asm("neg.f32 %0, %1;" : "=f"(_neg_497) : "f"(_abs_498)); + float _exp_496 = __nv_expf(_neg_497); + float2 rp_e_12 = make_float2(_exp_493, _exp_496); float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_467 = approx_rcp(rp_den_12.x); - float _rcp_468 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_469 = make_float2(_rcp_467, _rcp_468); + float _rcp_499 = approx_rcp(rp_den_12.x); + float _rcp_500 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_501 = make_float2(_rcp_499, _rcp_500); float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_501)); float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_470 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_502 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_470)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_502)); u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_471 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_503 = make_float2(gacc[4], gacc[4]); float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_471), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_474; - asm("abs.f32 %0, %1;" : "=f"(_abs_474) : "f"(rp_z_13.x)); - float _neg_473; - asm("neg.f32 %0, %1;" : "=f"(_neg_473) : "f"(_abs_474)); - float _exp_472 = __nv_expf(_neg_473); - float _abs_477; - asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(rp_z_13.y)); - float _neg_476; - asm("neg.f32 %0, %1;" : "=f"(_neg_476) : "f"(_abs_477)); - float _exp_475 = __nv_expf(_neg_476); - float2 rp_e_13 = make_float2(_exp_472, _exp_475); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_503), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_506; + asm("abs.f32 %0, %1;" : "=f"(_abs_506) : "f"(rp_z_13.x)); + float _neg_505; + asm("neg.f32 %0, %1;" : "=f"(_neg_505) : "f"(_abs_506)); + float _exp_504 = __nv_expf(_neg_505); + float _abs_509; + asm("abs.f32 %0, %1;" : "=f"(_abs_509) : "f"(rp_z_13.y)); + float _neg_508; + asm("neg.f32 %0, %1;" : "=f"(_neg_508) : "f"(_abs_509)); + float _exp_507 = __nv_expf(_neg_508); + float2 rp_e_13 = make_float2(_exp_504, _exp_507); float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_478 = approx_rcp(rp_den_13.x); - float _rcp_479 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_480 = make_float2(_rcp_478, _rcp_479); + float _rcp_510 = approx_rcp(rp_den_13.x); + float _rcp_511 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_512 = make_float2(_rcp_510, _rcp_511); float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_480)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_512)); float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_481 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_513 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_481)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_513)); u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_482 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_514 = make_float2(gacc[4], gacc[4]); float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_482), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_485; - asm("abs.f32 %0, %1;" : "=f"(_abs_485) : "f"(rp_z_14.x)); - float _neg_484; - asm("neg.f32 %0, %1;" : "=f"(_neg_484) : "f"(_abs_485)); - float _exp_483 = __nv_expf(_neg_484); - float _abs_488; - asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(rp_z_14.y)); - float _neg_487; - asm("neg.f32 %0, %1;" : "=f"(_neg_487) : "f"(_abs_488)); - float _exp_486 = __nv_expf(_neg_487); - float2 rp_e_14 = make_float2(_exp_483, _exp_486); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_514), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_517; + asm("abs.f32 %0, %1;" : "=f"(_abs_517) : "f"(rp_z_14.x)); + float _neg_516; + asm("neg.f32 %0, %1;" : "=f"(_neg_516) : "f"(_abs_517)); + float _exp_515 = __nv_expf(_neg_516); + float _abs_520; + asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_14.y)); + float _neg_519; + asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); + float _exp_518 = __nv_expf(_neg_519); + float2 rp_e_14 = make_float2(_exp_515, _exp_518); float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_489 = approx_rcp(rp_den_14.x); - float _rcp_490 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_491 = make_float2(_rcp_489, _rcp_490); + float _rcp_521 = approx_rcp(rp_den_14.x); + float _rcp_522 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_523 = make_float2(_rcp_521, _rcp_522); float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_491)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_492 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_524 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_492)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_524)); u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -5668,54 +5251,54 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_493; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_493) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_493; + uint32_t _prmt_525; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_525) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_525; int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_494 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_494)); - *(reinterpret_cast(_gst_494) + (0)) = pq_12[0]; + auto _gst_526 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_526)); + *(reinterpret_cast(_gst_526) + (0)) = pq_12[0]; } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_495; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_495) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_495; + uint32_t _prmt_527; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_527) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_527; int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_496 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_496)); - *(reinterpret_cast(_gst_496) + (0)) = pq_13[0]; + auto _gst_528 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_528)); + *(reinterpret_cast(_gst_528) + (0)) = pq_13[0]; } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_497; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_497) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_497; + uint32_t _prmt_529; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_529) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_529; int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_498 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_498)); - *(reinterpret_cast(_gst_498) + (0)) = pq_14[0]; + auto _gst_530 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_530)); + *(reinterpret_cast(_gst_530) + (0)) = pq_14[0]; } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_499; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_499) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_499; + uint32_t _prmt_531; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_531) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_531; int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_500 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_500)); - *(reinterpret_cast(_gst_500) + (0)) = pq_15[0]; + auto _gst_532 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_532)); + *(reinterpret_cast(_gst_532) + (0)) = pq_15[0]; } } int t_a_24 = 8 * th; @@ -5780,47 +5363,47 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_501; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_501) : "f"(s_a_2), "f"(nm_t_5)); + float _sub_533; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_533) : "f"(s_a_2), "f"(nm_t_5)); float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_501), "f"(pa_2)); - float _sub_502; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_502) : "f"(pa_2), "f"(nm_t_5)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_533), "f"(pa_2)); + float _sub_534; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_534) : "f"(pa_2), "f"(nm_t_5)); float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_502), "f"(s_a_2)); - float _abs_504; - asm("abs.f32 %0, %1;" : "=f"(_abs_504) : "f"(s_a_2)); - float _abs_505; - asm("abs.f32 %0, %1;" : "=f"(_abs_505) : "f"(pa_2)); - uint32_t _setp_503_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_503_bits) : "f"(_abs_504), "f"(_abs_505)); - bool _setp_503 = _setp_503_bits != 0; - float nm_err_5 = ((_setp_503) ? nm_es_5 : nm_ex_5); - float _add_506; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_506) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_506; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_534), "f"(s_a_2)); + float _abs_536; + asm("abs.f32 %0, %1;" : "=f"(_abs_536) : "f"(s_a_2)); + float _abs_537; + asm("abs.f32 %0, %1;" : "=f"(_abs_537) : "f"(pa_2)); + uint32_t _setp_535_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_535_bits) : "f"(_abs_536), "f"(_abs_537)); + bool _setp_535 = _setp_535_bits != 0; + float nm_err_5 = ((_setp_535) ? nm_es_5 : nm_ex_5); + float _add_538; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_538) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_538; s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_507; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_507) : "f"(s_b_2), "f"(nm_t_6)); + float _sub_539; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_539) : "f"(s_b_2), "f"(nm_t_6)); float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_507), "f"(pb_2)); - float _sub_508; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_508) : "f"(pb_2), "f"(nm_t_6)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_539), "f"(pb_2)); + float _sub_540; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_540) : "f"(pb_2), "f"(nm_t_6)); float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_508), "f"(s_b_2)); - float _abs_510; - asm("abs.f32 %0, %1;" : "=f"(_abs_510) : "f"(s_b_2)); - float _abs_511; - asm("abs.f32 %0, %1;" : "=f"(_abs_511) : "f"(pb_2)); - uint32_t _setp_509_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_509_bits) : "f"(_abs_510), "f"(_abs_511)); - bool _setp_509 = _setp_509_bits != 0; - float nm_err_6 = ((_setp_509) ? nm_es_6 : nm_ex_6); - float _add_512; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_512) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_512; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_540), "f"(s_b_2)); + float _abs_542; + asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(s_b_2)); + float _abs_543; + asm("abs.f32 %0, %1;" : "=f"(_abs_543) : "f"(pb_2)); + uint32_t _setp_541_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_541_bits) : "f"(_abs_542), "f"(_abs_543)); + bool _setp_541 = _setp_541_bits != 0; + float nm_err_6 = ((_setp_541) ? nm_es_6 : nm_ex_6); + float _add_544; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_544) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_544; s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -5859,60 +5442,60 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_514; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_514) : "f"(ts_s), "f"(ts_bb)); - float _sub_513; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_513) : "f"(s_a_3), "f"(_sub_514)); - float _sub_515; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_515) : "f"(other[0]), "f"(ts_bb)); + float _sub_546; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_546) : "f"(ts_s), "f"(ts_bb)); + float _sub_545; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_545) : "f"(s_a_3), "f"(_sub_546)); + float _sub_547; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_547) : "f"(other[0]), "f"(ts_bb)); float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_513), "f"(_sub_515)); - float _add_516; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_516) : "f"(c_a_3), "f"(other[1])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_545), "f"(_sub_547)); + float _add_548; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_548) : "f"(c_a_3), "f"(other[1])); float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_516), "f"(ts_e)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_548), "f"(ts_e)); float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_518; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_518) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_517; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_517) : "f"(ts_s), "f"(_sub_518)); - float _sub_519; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_519) : "f"(pm_c), "f"(ts_bb_0)); + float _sub_550; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_550) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_549; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_549) : "f"(ts_s), "f"(_sub_550)); + float _sub_551; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_551) : "f"(pm_c), "f"(ts_bb_0)); float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_517), "f"(_sub_519)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_549), "f"(_sub_551)); s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_521; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_521) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_520; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_520) : "f"(s_b_3), "f"(_sub_521)); - float _sub_522; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_522) : "f"(other[2]), "f"(ts_bb_1)); + float _sub_553; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_553) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_552; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_552) : "f"(s_b_3), "f"(_sub_553)); + float _sub_554; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_554) : "f"(other[2]), "f"(ts_bb_1)); float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_520), "f"(_sub_522)); - float _add_523; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_523) : "f"(c_b_3), "f"(other[3])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_552), "f"(_sub_554)); + float _add_555; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_555) : "f"(c_b_3), "f"(other[3])); float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_523), "f"(ts_e_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_555), "f"(ts_e_1)); float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_525; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_525) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_524; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_524) : "f"(ts_s_1), "f"(_sub_525)); - float _sub_526; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_526) : "f"(pm_c_0), "f"(ts_bb_2)); + float _sub_557; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_557) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_556; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_556) : "f"(ts_s_1), "f"(_sub_557)); + float _sub_558; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_558) : "f"(pm_c_0), "f"(ts_bb_2)); float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_524), "f"(_sub_526)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_556), "f"(_sub_558)); s_b_3 = ts_s_2; c_b_3 = ts_e_2; gacc[0] = s_a_3; @@ -5930,10 +5513,10 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM } if ((lane_1 & 3) == 0) { int e_star = lane_1 >> 2 & 7; - pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_1706; - pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1484; + pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_2; + pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1; pool_f32[28160 + qd * 64 + 32 + 8 * th + e_star] = v1; - pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_1241; + pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_0; } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync 1, 256;" ::: "memory"); @@ -6023,41 +5606,41 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + for (int r_284 = 0; r_284 < 16; r_284++) { + int t_3 = r_284; + if (t_3 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_3 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_3 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_284 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6129,41 +5712,41 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_24 = 16 + r_348; - if (t_24 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + for (int r_285 = 0; r_285 < 16; r_285++) { + int t_4 = 16 + r_285; + if (t_4 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_4 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_4 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_285 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6279,41 +5862,41 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_25 = 32 + r_349; - if (t_25 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + for (int r_286 = 0; r_286 < 16; r_286++) { + int t_5 = 32 + r_286; + if (t_5 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_5 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_5 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_286 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6461,41 +6044,41 @@ kernel_cake_kda_det_train_b16a0c1729083004fe80(const __grid_constant__ CUtensorM z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_26 = 48 + r_350; - if (t_26 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + for (int r_287 = 0; r_287 < 16; r_287++) { + int t_6 = 48 + r_287; + if (t_6 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_6 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_6 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_287 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu index bc7a0fd7ea4..589b41afc9f 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_c27b09eb5f5881be2969_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2246,15 +2269,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2275,15 +2304,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2304,15 +2339,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2333,15 +2374,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2502,15 +2573,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2531,15 +2608,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2560,15 +2643,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2589,15 +2678,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2636,15 +2731,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2665,15 +2766,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2694,15 +2801,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2723,15 +2836,21 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2802,37 +2921,37 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2965,19 +3084,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; - float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0) + dgf_0; + float qe = q_t0_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr)) + dgf_0; dgo[0] = dg_o_0; int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_248 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_249 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -3003,19 +3127,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_1 = dk2_1 * bt_t_1; float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; - float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1) + dgf_1; + float qe_0 = q_t0_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0)) + dgf_1; dgo[1] = dg_o_1; int qoff_1 = t_1 * qrs; if (t_1 < clen) { { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_250 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_251 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3041,19 +3170,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; - float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2) + dgf_2; + float qe_1 = q_t0_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1)) + dgf_2; dgo[2] = dg_o_2; int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_252 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_253 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3079,19 +3213,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; - float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3) + dgf_3; + float qe_2 = q_t0_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2)) + dgf_3; dgo[3] = dg_o_3; int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_254 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_255 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3117,19 +3256,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; - float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4) + dgf_4; + float qe_3 = q_t0_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3)) + dgf_4; dgo[4] = dg_o_4; int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_256 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_257 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3155,19 +3299,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; - float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5) + dgf_5; + float qe_4 = q_t0_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4)) + dgf_5; dgo[5] = dg_o_5; int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_258 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_259 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3193,19 +3342,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; - float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6) + dgf_6; + float qe_5 = q_t0_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5)) + dgf_6; dgo[6] = dg_o_6; int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_260 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_261 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3231,19 +3385,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; - float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7) + dgf_7; + float qe_6 = q_t0_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6)) + dgf_7; dgo[7] = dg_o_7; int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_262 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_263 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_7; } } float s_run[1]; @@ -3272,31 +3431,31 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3355,19 +3514,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; - float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8) + dgf_8; + float qe_7 = q_t0_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7)) + dgf_8; dgo_0[0] = dg_o_8; int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_273 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_274 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3393,19 +3557,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; - float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9) + dgf_9; + float qe_8 = q_t0_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8)) + dgf_9; dgo_0[1] = dg_o_9; int qoff_9 = t_9 * qrs; if (t_9 < clen) { { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_275 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_276 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3431,19 +3600,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; - float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10) + dgf_10; + float qe_9 = q_t0_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9)) + dgf_10; dgo_0[2] = dg_o_10; int qoff_10 = t_10 * qrs; if (t_10 < clen) { { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_277 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_278 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3469,19 +3643,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; - float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11) + dgf_11; + float qe_10 = q_t0_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10)) + dgf_11; dgo_0[3] = dg_o_11; int qoff_11 = t_11 * qrs; if (t_11 < clen) { { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_279 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_280 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3507,19 +3686,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; - float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12) + dgf_12; + float qe_11 = q_t0_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11)) + dgf_12; dgo_0[4] = dg_o_12; int qoff_12 = t_12 * qrs; if (t_12 < clen) { { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_281 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_282 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3545,19 +3729,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; - float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13) + dgf_13; + float qe_12 = q_t0_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12)) + dgf_13; dgo_0[5] = dg_o_13; int qoff_13 = t_13 * qrs; if (t_13 < clen) { { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_283 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_284 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3583,19 +3772,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; - float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14) + dgf_14; + float qe_13 = q_t0_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13)) + dgf_14; dgo_0[6] = dg_o_14; int qoff_14 = t_14 * qrs; if (t_14 < clen) { { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_285 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_286 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3621,19 +3815,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; - float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15) + dgf_15; + float qe_14 = q_t0_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14)) + dgf_15; dgo_0[7] = dg_o_15; int qoff_15 = t_15 * qrs; if (t_15 < clen) { { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_287 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_288 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_15; } } float s_run_0[1]; @@ -3662,36 +3861,36 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_289; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_290; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_291; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_292; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_293; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_294; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -3709,9 +3908,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_0 = offset + sfx[0]; if (t_16 < clen) { { - auto _gst_267 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_0; + auto _gst_299 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_0; } } int t_17 = 48 + 8 * th + 1; @@ -3719,9 +3918,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_1 = offset + sfx[1]; if (t_17 < clen) { { - auto _gst_268 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1; + auto _gst_300 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1; } } int t_18 = 48 + 8 * th + 2; @@ -3729,9 +3928,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_2 = offset + sfx[2]; if (t_18 < clen) { { - auto _gst_269 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_2; + auto _gst_301 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_2; } } int t_19 = 48 + 8 * th + 3; @@ -3739,9 +3938,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_3 = offset + sfx[3]; if (t_19 < clen) { { - auto _gst_270 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_3; + auto _gst_302 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_3; } } int t_20 = 48 + 8 * th + 4; @@ -3749,9 +3948,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_4 = offset + sfx[4]; if (t_20 < clen) { { - auto _gst_271 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_4; + auto _gst_303 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_4; } } int t_21 = 48 + 8 * th + 5; @@ -3759,9 +3958,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_5 = offset + sfx[5]; if (t_21 < clen) { { - auto _gst_272 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_5; + auto _gst_304 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_5; } } int t_22 = 48 + 8 * th + 6; @@ -3769,9 +3968,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_6 = offset + sfx[6]; if (t_22 < clen) { { - auto _gst_273 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_6; + auto _gst_305 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_6; } } int t_23 = 48 + 8 * th + 7; @@ -3779,9 +3978,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_7 = offset + sfx[7]; if (t_23 < clen) { { - auto _gst_274 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_7; + auto _gst_306 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3799,9 +3998,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_8 = offset_0 + sfx_0[0]; if (t_24 < clen) { { - auto _gst_275 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_8; + auto _gst_307 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_8; } } int t_25 = 32 + 8 * th + 1; @@ -3809,9 +4008,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_9 = offset_0 + sfx_0[1]; if (t_25 < clen) { { - auto _gst_276 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_9; + auto _gst_308 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_9; } } int t_26 = 32 + 8 * th + 2; @@ -3819,9 +4018,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_10 = offset_0 + sfx_0[2]; if (t_26 < clen) { { - auto _gst_277 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_10; + auto _gst_309 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_10; } } int t_27 = 32 + 8 * th + 3; @@ -3829,9 +4028,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_11 = offset_0 + sfx_0[3]; if (t_27 < clen) { { - auto _gst_278 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_11; + auto _gst_310 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_11; } } int t_28 = 32 + 8 * th + 4; @@ -3839,9 +4038,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_12 = offset_0 + sfx_0[4]; if (t_28 < clen) { { - auto _gst_279 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_12; + auto _gst_311 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_12; } } int t_29 = 32 + 8 * th + 5; @@ -3849,9 +4048,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_13 = offset_0 + sfx_0[5]; if (t_29 < clen) { { - auto _gst_280 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_13; + auto _gst_312 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_13; } } int t_30 = 32 + 8 * th + 6; @@ -3859,9 +4058,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_14 = offset_0 + sfx_0[6]; if (t_30 < clen) { { - auto _gst_281 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_14; + auto _gst_313 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_14; } } int t_31 = 32 + 8 * th + 7; @@ -3869,15 +4068,15 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_15 = offset_0 + sfx_0[7]; if (t_31 < clen) { { - auto _gst_282 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_15; + auto _gst_314 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3936,19 +4135,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; - float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16) + dgf_16; + float qe_15 = q_t0_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15)) + dgf_16; dgo_1[0] = dg_o_16; int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_284 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_316 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_285 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_317 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_33 = 16 + 8 * th + 1; @@ -3974,19 +4178,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; - float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17) + dgf_17; + float qe_16 = q_t0_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16)) + dgf_17; dgo_1[1] = dg_o_17; int qoff_17 = t_33 * qrs; if (t_33 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_34 = 16 + 8 * th + 2; @@ -4012,19 +4221,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; - float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18) + dgf_18; + float qe_17 = q_t0_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17)) + dgf_18; dgo_1[2] = dg_o_18; int qoff_18 = t_34 * qrs; if (t_34 < clen) { { - auto _gst_288 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_320 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_289 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_321 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_35 = 16 + 8 * th + 3; @@ -4050,19 +4264,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; - float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19) + dgf_19; + float qe_18 = q_t0_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18)) + dgf_19; dgo_1[3] = dg_o_19; int qoff_19 = t_35 * qrs; if (t_35 < clen) { { - auto _gst_290 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_322 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_291 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_323 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_36 = 16 + 8 * th + 4; @@ -4088,19 +4307,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; - float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20) + dgf_20; + float qe_19 = q_t0_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19)) + dgf_20; dgo_1[4] = dg_o_20; int qoff_20 = t_36 * qrs; if (t_36 < clen) { { - auto _gst_292 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_324 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_293 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_325 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_37 = 16 + 8 * th + 5; @@ -4126,19 +4350,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; - float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21) + dgf_21; + float qe_20 = q_t0_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20)) + dgf_21; dgo_1[5] = dg_o_21; int qoff_21 = t_37 * qrs; if (t_37 < clen) { { - auto _gst_294 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_326 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_295 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_327 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_38 = 16 + 8 * th + 6; @@ -4164,19 +4393,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; - float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22) + dgf_22; + float qe_21 = q_t0_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21)) + dgf_22; dgo_1[6] = dg_o_22; int qoff_22 = t_38 * qrs; if (t_38 < clen) { { - auto _gst_296 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_328 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_297 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_329 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_39 = 16 + 8 * th + 7; @@ -4202,19 +4436,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; - float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23) + dgf_23; + float qe_22 = q_t0_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22)) + dgf_23; dgo_1[7] = dg_o_23; int qoff_23 = t_39 * qrs; if (t_39 < clen) { { - auto _gst_298 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_330 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_299 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_331 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_23; } } float s_run_1[1]; @@ -4243,37 +4482,37 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_332; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_333; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_334; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_335; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_336; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_337; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4291,9 +4530,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_16 = offset_1 + sfx_1[0]; if (t_40 < clen) { { - auto _gst_310 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_16; + auto _gst_342 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_16; } } int t_41 = 16 + 8 * th + 1; @@ -4301,9 +4540,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_17 = offset_1 + sfx_1[1]; if (t_41 < clen) { { - auto _gst_311 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_17; + auto _gst_343 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_17; } } int t_42 = 16 + 8 * th + 2; @@ -4311,9 +4550,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_18 = offset_1 + sfx_1[2]; if (t_42 < clen) { { - auto _gst_312 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_18; + auto _gst_344 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_18; } } int t_43 = 16 + 8 * th + 3; @@ -4321,9 +4560,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_19 = offset_1 + sfx_1[3]; if (t_43 < clen) { { - auto _gst_313 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_19; + auto _gst_345 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_19; } } int t_44 = 16 + 8 * th + 4; @@ -4331,9 +4570,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_20 = offset_1 + sfx_1[4]; if (t_44 < clen) { { - auto _gst_314 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_20; + auto _gst_346 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_20; } } int t_45 = 16 + 8 * th + 5; @@ -4341,9 +4580,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_21 = offset_1 + sfx_1[5]; if (t_45 < clen) { { - auto _gst_315 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_21; + auto _gst_347 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_21; } } int t_46 = 16 + 8 * th + 6; @@ -4351,9 +4590,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_22 = offset_1 + sfx_1[6]; if (t_46 < clen) { { - auto _gst_316 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_22; + auto _gst_348 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_22; } } int t_47 = 16 + 8 * th + 7; @@ -4361,15 +4600,15 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_23 = offset_1 + sfx_1[7]; if (t_47 < clen) { { - auto _gst_317 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_23; + auto _gst_349 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4422,19 +4661,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; - float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24) + dgf_24; + float qe_23 = q_t0_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23)) + dgf_24; dgo_2[0] = dg_o_24; int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_319 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_351 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_320 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_352 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_49 = 8 * th + 1; @@ -4460,19 +4704,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_25 = dk2_25 * bt_t_25; float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; - float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25) + dgf_25; + float qe_24 = q_t0_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24)) + dgf_25; dgo_2[1] = dg_o_25; int qoff_25 = t_49 * qrs; if (t_49 < clen) { { - auto _gst_321 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_353 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_322 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_354 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_50 = 8 * th + 2; @@ -4498,19 +4747,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_26 = dk2_26 * bt_t_26; float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; - float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26) + dgf_26; + float qe_25 = q_t0_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25)) + dgf_26; dgo_2[2] = dg_o_26; int qoff_26 = t_50 * qrs; if (t_50 < clen) { { - auto _gst_323 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_355 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_324 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_356 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_51 = 8 * th + 3; @@ -4536,19 +4790,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_27 = dk2_27 * bt_t_27; float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; - float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27) + dgf_27; + float qe_26 = q_t0_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26)) + dgf_27; dgo_2[3] = dg_o_27; int qoff_27 = t_51 * qrs; if (t_51 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_52 = 8 * th + 4; @@ -4574,19 +4833,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_28 = dk2_28 * bt_t_28; float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; - float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28) + dgf_28; + float qe_27 = q_t0_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27)) + dgf_28; dgo_2[4] = dg_o_28; int qoff_28 = t_52 * qrs; if (t_52 < clen) { { - auto _gst_327 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_359 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_328 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_360 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_53 = 8 * th + 5; @@ -4612,19 +4876,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_29 = dk2_29 * bt_t_29; float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; - float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29) + dgf_29; + float qe_28 = q_t0_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28)) + dgf_29; dgo_2[5] = dg_o_29; int qoff_29 = t_53 * qrs; if (t_53 < clen) { { - auto _gst_329 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_361 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_330 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_362 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_54 = 8 * th + 6; @@ -4650,19 +4919,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_30 = dk2_30 * bt_t_30; float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; - float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30) + dgf_30; + float qe_29 = q_t0_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29)) + dgf_30; dgo_2[6] = dg_o_30; int qoff_30 = t_54 * qrs; if (t_54 < clen) { { - auto _gst_331 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_363 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_332 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_364 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_55 = 8 * th + 7; @@ -4688,19 +4962,24 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dk2b_31 = dk2_31 * bt_t_31; float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; - float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31) + dgf_31; + float qe_30 = q_t0_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30)) + dgf_31; dgo_2[7] = dg_o_31; int qoff_31 = t_55 * qrs; if (t_55 < clen) { { - auto _gst_333 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_365 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_334 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_366 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_31; } } float s_run_2[1]; @@ -4729,35 +5008,35 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_367; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_368; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_369; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_370; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_371; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_372; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_2) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4775,9 +5054,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_24 = offset_2 + sfx_2[0]; if (t_56 < clen) { { - auto _gst_345 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_24; + auto _gst_377 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_24; } } int t_57 = 8 * th + 1; @@ -4785,9 +5064,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_25 = offset_2 + sfx_2[1]; if (t_57 < clen) { { - auto _gst_346 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_25; + auto _gst_378 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_25; } } int t_58 = 8 * th + 2; @@ -4795,9 +5074,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_26 = offset_2 + sfx_2[2]; if (t_58 < clen) { { - auto _gst_347 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_26; + auto _gst_379 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_26; } } int t_59 = 8 * th + 3; @@ -4805,9 +5084,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_27 = offset_2 + sfx_2[3]; if (t_59 < clen) { { - auto _gst_348 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_27; + auto _gst_380 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_27; } } int t_60 = 8 * th + 4; @@ -4815,9 +5094,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_28 = offset_2 + sfx_2[4]; if (t_60 < clen) { { - auto _gst_349 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_28; + auto _gst_381 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_28; } } int t_61 = 8 * th + 5; @@ -4825,9 +5104,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_29 = offset_2 + sfx_2[5]; if (t_61 < clen) { { - auto _gst_350 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_29; + auto _gst_382 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_29; } } int t_62 = 8 * th + 6; @@ -4835,9 +5114,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_30 = offset_2 + sfx_2[6]; if (t_62 < clen) { { - auto _gst_351 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_30; + auto _gst_383 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_30; } } int t_63 = 8 * th + 7; @@ -4845,9 +5124,9 @@ kernel_cake_kda_det_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorM float dg_31 = offset_2 + sfx_2[7]; if (t_63 < clen) { { - auto _gst_352 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_31; + auto _gst_384 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_31; } } off_acc[0] = off_acc[0] + t_hi_2 + t_lo_2; diff --git a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu index 8a6efdd1ecf..e9d1eba4441 100644 --- a/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu +++ b/flashinfer/experimental/cake_kda_det_train/csrc/cake_kda_det_train_ea14e16d1fc71364ba03_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -199,6 +200,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2267,12 +2284,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_0); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_0 : __fmaf_rn(dl, _rcp_170, e2_0)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2296,12 +2319,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_1); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_1, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_1, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_1 : __fmaf_rn(dl_1, _rcp_174, e2_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2325,12 +2354,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_2); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_2, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_2, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_2 : __fmaf_rn(dl_3, _rcp_178, e2_2)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2354,12 +2389,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_3); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_3, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_3, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_3 : __fmaf_rn(dl_5, _rcp_182, e2_3)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2395,12 +2436,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2420,12 +2467,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2445,12 +2498,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2470,12 +2529,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2523,12 +2588,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2552,12 +2623,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_1 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2581,12 +2658,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_2 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2610,12 +2693,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_3 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2657,12 +2746,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_4 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2686,12 +2781,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_5 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2715,12 +2816,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_6 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2744,12 +2851,18 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 dn2_7 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2820,37 +2933,37 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float2 d2_56 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_56.x); float e1_12 = approx_exp2(d2_56.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_57.x); float e1_13 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_58.x); float e1_14 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_59.x); float e1_15 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2862,29 +2975,29 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2892,29 +3005,29 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2933,409 +3046,145 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float r_247[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_247[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 4096 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 4096 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); float gn = r_247[0]; - float gc_888 = gn - 21.640425f; + float gc_0 = gn - 21.640425f; int eb = 98304 + th * -32768; int ew = eb >> 2; - int qbase_889 = pool_f32_addr + (unsigned int)(eb + 12288 + dh_lane * 1024); + int qbase_0 = pool_f32_addr + (unsigned int)(eb + 12288 + dh_lane * 1024); int sb8 = 32 + 8 * th; int gb8 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8 * 128); int kb8 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8 * 128); - float dbp[8]; - float _tmem_load_7[8]; - tmem_ld_x8(&_tmem_load_7[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float _tmem_load_8[8]; - tmem_ld_x8(&_tmem_load_8[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float _tmem_load_9[8]; - tmem_ld_x8(&_tmem_load_9[0], taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)); - float _tmem_load_10[8]; - tmem_ld_x8(&_tmem_load_10[0], taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th)); - float _tmem_load_11[8]; - tmem_ld_x8(&_tmem_load_11[0], taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th)); - float btv[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[0])), "=r"(*reinterpret_cast(&btv[(0) + 1])), "=r"(*reinterpret_cast(&btv[(0) + 2])), "=r"(*reinterpret_cast(&btv[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[4])), "=r"(*reinterpret_cast(&btv[(4) + 1])), "=r"(*reinterpret_cast(&btv[(4) + 2])), "=r"(*reinterpret_cast(&btv[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int c_qa = 128 + 8 * th; + int c_ka = 144 + 8 * th; + int c_qd = 192 + 8 * th; + int c_kd = 208 + 8 * th; + int c_dk = 32 + 8 * th; + int c_bp = c_ka; float s_run[1]; s_run[0] = 0.0f; - int t_1_0 = 32 + 8 * th + 7; - float r_248[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_0 = r_248[0]; - unsigned int r_249[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_0 = __uint_as_float(r_249[0] << 16); - unsigned int r_250[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_0 = __uint_as_float(r_250[0] << 16); - float q_t_2_0 = ((t_1_0 >= clen) ? 0.0f : q_t_1_0); - float dqf_1_0 = pool_f32[ew + 896 + d_lane]; - float dkf_1_0 = pool_f32[ew + 1920 + d_lane]; - float dgf_1_0 = pool_f32[ew + 2944 + d_lane]; - float bt_t_1_0 = btv[7]; - float e_off_1_0 = approx_exp2(g_t_1_0 - gn); - float e_dg_1_0 = approx_exp2(g_t_1_0 - gc_888); - float dq2_1_0 = __fmaf_rn(_tmem_load_7[7], e_off_1_0, _tmem_load_9[7] * e_dg_1_0); - float dk2_1_0 = __fmaf_rn(_tmem_load_8[7], e_off_1_0, _tmem_load_10[7] * e_dg_1_0); - dbp[7] = dk2_1_0 * k_t_1_0; - float dk2b_1_0 = dk2_1_0 * bt_t_1_0; - float dq_o_1_0 = dq2_1_0 + dqf_1_0; - float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; - s_run[0] = s_run[0] + dg_o_1_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 7)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_0 = t_1_0 * qrs; - if (t_1_0 < clen) { - { - auto _gst_216 = dq_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_1_0; - } - { - auto _gst_217 = dk_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_1_0; - } - } - int t_1_1 = 32 + 8 * th + 6; - float r_251[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_1 = r_251[0]; - unsigned int r_252[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((kb8 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_1 = __uint_as_float(r_252[0] << 16); - unsigned int r_253[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_1 = __uint_as_float(r_253[0] << 16); - float q_t_2_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_1); - float dqf_1_1 = pool_f32[ew + 768 + d_lane]; - float dkf_1_1 = pool_f32[ew + 1792 + d_lane]; - float dgf_1_1 = pool_f32[ew + 2816 + d_lane]; - float bt_t_1_1 = btv[6]; - float e_off_1_1 = approx_exp2(g_t_1_1 - gn); - float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_888); - float dq2_1_1 = __fmaf_rn(_tmem_load_7[6], e_off_1_1, _tmem_load_9[6] * e_dg_1_1); - float dk2_1_1 = __fmaf_rn(_tmem_load_8[6], e_off_1_1, _tmem_load_10[6] * e_dg_1_1); - dbp[6] = dk2_1_1 * k_t_1_1; - float dk2b_1_1 = dk2_1_1 * bt_t_1_1; - float dq_o_1_1 = dq2_1_1 + dqf_1_1; - float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; - s_run[0] = s_run[0] + dg_o_1_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 6)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_1 = t_1_1 * qrs; - if (t_1_1 < clen) { - { - auto _gst_218 = dq_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1_1; - } - { - auto _gst_219 = dk_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1_1; - } - } - int t_1_2 = 32 + 8 * th + 5; - float r_254[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_2 = r_254[0]; - unsigned int r_255[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((kb8 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_2 = __uint_as_float(r_255[0] << 16); - unsigned int r_256[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_2 = __uint_as_float(r_256[0] << 16); - float q_t_2_2 = ((t_1_2 >= clen) ? 0.0f : q_t_1_2); - float dqf_1_2 = pool_f32[ew + 640 + d_lane]; - float dkf_1_2 = pool_f32[ew + 1664 + d_lane]; - float dgf_1_2 = pool_f32[ew + 2688 + d_lane]; - float bt_t_1_2 = btv[5]; - float e_off_1_2 = approx_exp2(g_t_1_2 - gn); - float e_dg_1_2 = approx_exp2(g_t_1_2 - gc_888); - float dq2_1_2 = __fmaf_rn(_tmem_load_7[5], e_off_1_2, _tmem_load_9[5] * e_dg_1_2); - float dk2_1_2 = __fmaf_rn(_tmem_load_8[5], e_off_1_2, _tmem_load_10[5] * e_dg_1_2); - dbp[5] = dk2_1_2 * k_t_1_2; - float dk2b_1_2 = dk2_1_2 * bt_t_1_2; - float dq_o_1_2 = dq2_1_2 + dqf_1_2; - float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; - s_run[0] = s_run[0] + dg_o_1_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 5)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_2 = t_1_2 * qrs; - if (t_1_2 < clen) { - { - auto _gst_220 = dq_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1_2; - } - { - auto _gst_221 = dk_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1_2; - } - } - int t_1_3 = 32 + 8 * th + 4; - float r_257[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_3 = r_257[0]; - unsigned int r_258[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((kb8 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_3 = __uint_as_float(r_258[0] << 16); - unsigned int r_259[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_3 = __uint_as_float(r_259[0] << 16); - float q_t_2_3 = ((t_1_3 >= clen) ? 0.0f : q_t_1_3); - float dqf_1_3 = pool_f32[ew + 512 + d_lane]; - float dkf_1_3 = pool_f32[ew + 1536 + d_lane]; - float dgf_1_3 = pool_f32[ew + 2560 + d_lane]; - float bt_t_1_3 = btv[4]; - float e_off_1_3 = approx_exp2(g_t_1_3 - gn); - float e_dg_1_3 = approx_exp2(g_t_1_3 - gc_888); - float dq2_1_3 = __fmaf_rn(_tmem_load_7[4], e_off_1_3, _tmem_load_9[4] * e_dg_1_3); - float dk2_1_3 = __fmaf_rn(_tmem_load_8[4], e_off_1_3, _tmem_load_10[4] * e_dg_1_3); - dbp[4] = dk2_1_3 * k_t_1_3; - float dk2b_1_3 = dk2_1_3 * bt_t_1_3; - float dq_o_1_3 = dq2_1_3 + dqf_1_3; - float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; - s_run[0] = s_run[0] + dg_o_1_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 4)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_3 = t_1_3 * qrs; - if (t_1_3 < clen) { - { - auto _gst_222 = dq_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_1_3; - } - { - auto _gst_223 = dk_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_1_3; - } - } - int t_1_4 = 32 + 8 * th + 3; - float r_260[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_4 = r_260[0]; - unsigned int r_261[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((kb8 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_4 = __uint_as_float(r_261[0] << 16); - unsigned int r_262[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_4 = __uint_as_float(r_262[0] << 16); - float q_t_2_4 = ((t_1_4 >= clen) ? 0.0f : q_t_1_4); - float dqf_1_4 = pool_f32[ew + 384 + d_lane]; - float dkf_1_4 = pool_f32[ew + 1408 + d_lane]; - float dgf_1_4 = pool_f32[ew + 2432 + d_lane]; - float bt_t_1_4 = btv[3]; - float e_off_1_4 = approx_exp2(g_t_1_4 - gn); - float e_dg_1_4 = approx_exp2(g_t_1_4 - gc_888); - float dq2_1_4 = __fmaf_rn(_tmem_load_7[3], e_off_1_4, _tmem_load_9[3] * e_dg_1_4); - float dk2_1_4 = __fmaf_rn(_tmem_load_8[3], e_off_1_4, _tmem_load_10[3] * e_dg_1_4); - dbp[3] = dk2_1_4 * k_t_1_4; - float dk2b_1_4 = dk2_1_4 * bt_t_1_4; - float dq_o_1_4 = dq2_1_4 + dqf_1_4; - float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; - s_run[0] = s_run[0] + dg_o_1_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 3)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_4 = t_1_4 * qrs; - if (t_1_4 < clen) { - { - auto _gst_224 = dq_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_1_4; - } - { - auto _gst_225 = dk_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_1_4; - } - } - int t_1_5 = 32 + 8 * th + 2; - float r_263[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_5 = r_263[0]; - unsigned int r_264[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((kb8 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_5 = __uint_as_float(r_264[0] << 16); - unsigned int r_265[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_5 = __uint_as_float(r_265[0] << 16); - float q_t_2_5 = ((t_1_5 >= clen) ? 0.0f : q_t_1_5); - float dqf_1_5 = pool_f32[ew + 256 + d_lane]; - float dkf_1_5 = pool_f32[ew + 1280 + d_lane]; - float dgf_1_5 = pool_f32[ew + 2304 + d_lane]; - float bt_t_1_5 = btv[2]; - float e_off_1_5 = approx_exp2(g_t_1_5 - gn); - float e_dg_1_5 = approx_exp2(g_t_1_5 - gc_888); - float dq2_1_5 = __fmaf_rn(_tmem_load_7[2], e_off_1_5, _tmem_load_9[2] * e_dg_1_5); - float dk2_1_5 = __fmaf_rn(_tmem_load_8[2], e_off_1_5, _tmem_load_10[2] * e_dg_1_5); - dbp[2] = dk2_1_5 * k_t_1_5; - float dk2b_1_5 = dk2_1_5 * bt_t_1_5; - float dq_o_1_5 = dq2_1_5 + dqf_1_5; - float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; - s_run[0] = s_run[0] + dg_o_1_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 2)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_5 = t_1_5 * qrs; - if (t_1_5 < clen) { - { - auto _gst_226 = dq_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_1_5; - } - { - auto _gst_227 = dk_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_1_5; - } - } - int t_1_6 = 32 + 8 * th + 1; - float r_266[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_6 = r_266[0]; - unsigned int r_267[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_6 = __uint_as_float(r_267[0] << 16); - unsigned int r_268[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_6 = __uint_as_float(r_268[0] << 16); - float q_t_2_6 = ((t_1_6 >= clen) ? 0.0f : q_t_1_6); - float dqf_1_6 = pool_f32[ew + 128 + d_lane]; - float dkf_1_6 = pool_f32[ew + 1152 + d_lane]; - float dgf_1_6 = pool_f32[ew + 2176 + d_lane]; - float bt_t_1_6 = btv[1]; - float e_off_1_6 = approx_exp2(g_t_1_6 - gn); - float e_dg_1_6 = approx_exp2(g_t_1_6 - gc_888); - float dq2_1_6 = __fmaf_rn(_tmem_load_7[1], e_off_1_6, _tmem_load_9[1] * e_dg_1_6); - float dk2_1_6 = __fmaf_rn(_tmem_load_8[1], e_off_1_6, _tmem_load_10[1] * e_dg_1_6); - dbp[1] = dk2_1_6 * k_t_1_6; - float dk2b_1_6 = dk2_1_6 * bt_t_1_6; - float dq_o_1_6 = dq2_1_6 + dqf_1_6; - float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; - s_run[0] = s_run[0] + dg_o_1_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 1)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_6 = t_1_6 * qrs; - if (t_1_6 < clen) { - { - auto _gst_228 = dq_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_1_6; - } - { - auto _gst_229 = dk_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_1_6; - } - } - int t_1_7 = 32 + 8 * th; - float r_269[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_7 = r_269[0]; - unsigned int r_270[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((kb8 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_7 = __uint_as_float(r_270[0] << 16); - unsigned int r_271[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_7 = __uint_as_float(r_271[0] << 16); - float q_t_2_7 = ((t_1_7 >= clen) ? 0.0f : q_t_1_7); - float dqf_1_7 = pool_f32[ew + d_lane]; - float dkf_1_7 = pool_f32[ew + 1024 + d_lane]; - float dgf_1_7 = pool_f32[ew + 2048 + d_lane]; - float bt_t_1_7 = btv[0]; - float e_off_1_7 = approx_exp2(g_t_1_7 - gn); - float e_dg_1_7 = approx_exp2(g_t_1_7 - gc_888); - float dq2_1_7 = __fmaf_rn(_tmem_load_7[0], e_off_1_7, _tmem_load_9[0] * e_dg_1_7); - float dk2_1_7 = __fmaf_rn(_tmem_load_8[0], e_off_1_7, _tmem_load_10[0] * e_dg_1_7); - dbp[0] = dk2_1_7 * k_t_1_7; - float dk2b_1_7 = dk2_1_7 * bt_t_1_7; - float dq_o_1_7 = dq2_1_7 + dqf_1_7; - float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; - s_run[0] = s_run[0] + dg_o_1_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_7 = t_1_7 * qrs; - if (t_1_7 < clen) { - { - auto _gst_230 = dq_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_1_7; - } - { - auto _gst_231 = dk_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_1_7; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j = 7 - k; + float a_qd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j))); + float a_kd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd + j))); + float a_dk[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk + j))); + float a_qa[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa + j))); + float a_ka[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka + j))); + float btr[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8 + j)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8 + j; + float r_248[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_248[0]; + unsigned int r_249[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_249[0] << 16); + unsigned int r_250[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_250[0] << 16); + float q_m = ((t >= clen) ? 0.0f : q_t); + float dqf = pool_f32[ew + 128 * j + d_lane]; + float dkf = pool_f32[ew + (1024 + 128 * j) + d_lane]; + float dgf = pool_f32[ew + (2048 + 128 * j) + d_lane]; + float bt_t = btr[0]; + float e_dg = approx_exp2(g_t - gc_0); + float e_off = approx_exp2(g_t - gn); + float dq2 = __fmaf_rn(a_qa[0], e_off, a_qd[0] * e_dg); + float dk2 = __fmaf_rn(a_ka[0], e_off, a_kd[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + a_dk[0]; + float qe = q_m * e_dg; + float kp = k_t * bt_t; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, a_qd[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg) * a_kd[0]); + float dg_o = __fmaf_rn(q_m, dq2, __fmaf_rn(dk2b - a_dk[0], k_t, cr)) + dgf; + s_run[0] = s_run[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j)), "f"(s_run[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff = t * qrs; + if (t < clen) { + { + auto _gst_248 = dq_base + qoff; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o; + } + { + auto _gst_249 = dk_base + qoff; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o; + } } } float tot = s_run[0]; + float dbp[8]; + tmem_ld_x8(&dbp[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); int b4 = lane_1 >> 4 & 1; int b3 = lane_1 >> 3 & 1; int b2 = lane_1 >> 2 & 1; float v4[4]; - float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; - float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; - float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; - float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; - float v2_1044[2]; - float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; - float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; - float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_250; + float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_251; + float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_252; + float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_253; + float v2[2]; + float send2_0 = ((b3 == 0) ? v4[2] : v4[0]); + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send2_0, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_254; + float send2_1 = ((b3 == 0) ? v4[3] : v4[1]); + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send2_1, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_255; + float send1 = ((b2 == 0) ? v2[1] : v2[0]); + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send1, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_256; + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_257; + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_258; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3346,412 +3195,148 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM if (lane_1 == 17) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(dt_bias + (hv * 128 + 32 * qd)))); } mbarrier_wait(epi_full_addr + (th) * 8, 1); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_272[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1049 = r_272[0]; - float gc_1050 = gn_1049 - 21.640425f; - int eb_1051 = 98304 + th * -32768; - int ew_1052 = eb_1051 >> 2; - int qbase_1053 = pool_f32_addr + (unsigned int)(eb_1051 + 12288 + dh_lane * 1024); - int sb8_1054 = 48 + 8 * th; - int gb8_1055 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1054 * 128); - int kb8_1056 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1054 * 128); - float dbp_1057[8]; - float _tmem_load_12[8]; - tmem_ld_x8(&_tmem_load_12[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float _tmem_load_13[8]; - tmem_ld_x8(&_tmem_load_13[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float _tmem_load_14[8]; - tmem_ld_x8(&_tmem_load_14[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float _tmem_load_15[8]; - tmem_ld_x8(&_tmem_load_15[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float _tmem_load_16[8]; - tmem_ld_x8(&_tmem_load_16[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_1058[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[0])), "=r"(*reinterpret_cast(&btv_1058[(0) + 1])), "=r"(*reinterpret_cast(&btv_1058[(0) + 2])), "=r"(*reinterpret_cast(&btv_1058[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[4])), "=r"(*reinterpret_cast(&btv_1058[(4) + 1])), "=r"(*reinterpret_cast(&btv_1058[(4) + 2])), "=r"(*reinterpret_cast(&btv_1058[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1059[1]; - s_run_1059[0] = 0.0f; - int t_0 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew_1052 + 896 + d_lane]; - float dkf_0 = pool_f32[ew_1052 + 1920 + d_lane]; - float dgf_0 = pool_f32[ew_1052 + 2944 + d_lane]; - float bt_t_0 = btv_1058[7]; - float e_off_0 = approx_exp2(g_t_0 - gn_1049); - float e_dg_0 = approx_exp2(g_t_0 - gc_1050); - float dq2_0 = __fmaf_rn(_tmem_load_12[7], e_off_0, _tmem_load_14[7] * e_dg_0); - float dk2_0 = __fmaf_rn(_tmem_load_13[7], e_off_0, _tmem_load_15[7] * e_dg_0); - dbp_1057[7] = dk2_0 * k_t_0; - float dk2b_0 = dk2_0 * bt_t_0; - float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; - s_run_1059[0] = s_run_1059[0] + dg_o_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_0 = t_0 * qrs; - if (t_0 < clen) { - { - auto _gst_241 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_0; - } - { - auto _gst_242 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_0; - } - } - int t_1 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew_1052 + 768 + d_lane]; - float dkf_1 = pool_f32[ew_1052 + 1792 + d_lane]; - float dgf_1 = pool_f32[ew_1052 + 2816 + d_lane]; - float bt_t_1 = btv_1058[6]; - float e_off_1 = approx_exp2(g_t_1 - gn_1049); - float e_dg_1 = approx_exp2(g_t_1 - gc_1050); - float dq2_1 = __fmaf_rn(_tmem_load_12[6], e_off_1, _tmem_load_14[6] * e_dg_1); - float dk2_1 = __fmaf_rn(_tmem_load_13[6], e_off_1, _tmem_load_15[6] * e_dg_1); - dbp_1057[6] = dk2_1 * k_t_1; - float dk2b_1 = dk2_1 * bt_t_1; - float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + _tmem_load_16[6]; - float dg_o_1 = __fmaf_rn(q_t_1_9, dq2_1, (dk2b_1 - _tmem_load_16[6]) * k_t_1) + dgf_1; - s_run_1059[0] = s_run_1059[0] + dg_o_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1 = t_1 * qrs; - if (t_1 < clen) { - { - auto _gst_243 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_1; - } - { - auto _gst_244 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_1; - } - } - int t_2 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew_1052 + 640 + d_lane]; - float dkf_2 = pool_f32[ew_1052 + 1664 + d_lane]; - float dgf_2 = pool_f32[ew_1052 + 2688 + d_lane]; - float bt_t_2 = btv_1058[5]; - float e_off_2 = approx_exp2(g_t_2 - gn_1049); - float e_dg_2 = approx_exp2(g_t_2 - gc_1050); - float dq2_2 = __fmaf_rn(_tmem_load_12[5], e_off_2, _tmem_load_14[5] * e_dg_2); - float dk2_2 = __fmaf_rn(_tmem_load_13[5], e_off_2, _tmem_load_15[5] * e_dg_2); - dbp_1057[5] = dk2_2 * k_t_2; - float dk2b_2 = dk2_2 * bt_t_2; - float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[5]; - float dg_o_2 = __fmaf_rn(q_t_1_10, dq2_2, (dk2b_2 - _tmem_load_16[5]) * k_t_2) + dgf_2; - s_run_1059[0] = s_run_1059[0] + dg_o_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_2 = t_2 * qrs; - if (t_2 < clen) { - { - auto _gst_245 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_2; - } - { - auto _gst_246 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_2; - } - } - int t_3 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew_1052 + 512 + d_lane]; - float dkf_3 = pool_f32[ew_1052 + 1536 + d_lane]; - float dgf_3 = pool_f32[ew_1052 + 2560 + d_lane]; - float bt_t_3 = btv_1058[4]; - float e_off_3 = approx_exp2(g_t_3 - gn_1049); - float e_dg_3 = approx_exp2(g_t_3 - gc_1050); - float dq2_3 = __fmaf_rn(_tmem_load_12[4], e_off_3, _tmem_load_14[4] * e_dg_3); - float dk2_3 = __fmaf_rn(_tmem_load_13[4], e_off_3, _tmem_load_15[4] * e_dg_3); - dbp_1057[4] = dk2_3 * k_t_3; - float dk2b_3 = dk2_3 * bt_t_3; - float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[4]; - float dg_o_3 = __fmaf_rn(q_t_1_11, dq2_3, (dk2b_3 - _tmem_load_16[4]) * k_t_3) + dgf_3; - s_run_1059[0] = s_run_1059[0] + dg_o_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_3 = t_3 * qrs; - if (t_3 < clen) { - { - auto _gst_247 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_3; - } - { - auto _gst_248 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_3; - } - } - int t_4 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew_1052 + 384 + d_lane]; - float dkf_4 = pool_f32[ew_1052 + 1408 + d_lane]; - float dgf_4 = pool_f32[ew_1052 + 2432 + d_lane]; - float bt_t_4 = btv_1058[3]; - float e_off_4 = approx_exp2(g_t_4 - gn_1049); - float e_dg_4 = approx_exp2(g_t_4 - gc_1050); - float dq2_4 = __fmaf_rn(_tmem_load_12[3], e_off_4, _tmem_load_14[3] * e_dg_4); - float dk2_4 = __fmaf_rn(_tmem_load_13[3], e_off_4, _tmem_load_15[3] * e_dg_4); - dbp_1057[3] = dk2_4 * k_t_4; - float dk2b_4 = dk2_4 * bt_t_4; - float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[3]; - float dg_o_4 = __fmaf_rn(q_t_1_12, dq2_4, (dk2b_4 - _tmem_load_16[3]) * k_t_4) + dgf_4; - s_run_1059[0] = s_run_1059[0] + dg_o_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_4 = t_4 * qrs; - if (t_4 < clen) { - { - auto _gst_249 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_4; - } - { - auto _gst_250 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_4; - } - } - int t_5 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew_1052 + 256 + d_lane]; - float dkf_5 = pool_f32[ew_1052 + 1280 + d_lane]; - float dgf_5 = pool_f32[ew_1052 + 2304 + d_lane]; - float bt_t_5 = btv_1058[2]; - float e_off_5 = approx_exp2(g_t_5 - gn_1049); - float e_dg_5 = approx_exp2(g_t_5 - gc_1050); - float dq2_5 = __fmaf_rn(_tmem_load_12[2], e_off_5, _tmem_load_14[2] * e_dg_5); - float dk2_5 = __fmaf_rn(_tmem_load_13[2], e_off_5, _tmem_load_15[2] * e_dg_5); - dbp_1057[2] = dk2_5 * k_t_5; - float dk2b_5 = dk2_5 * bt_t_5; - float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[2]; - float dg_o_5 = __fmaf_rn(q_t_1_13, dq2_5, (dk2b_5 - _tmem_load_16[2]) * k_t_5) + dgf_5; - s_run_1059[0] = s_run_1059[0] + dg_o_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_5 = t_5 * qrs; - if (t_5 < clen) { - { - auto _gst_251 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_5; - } - { - auto _gst_252 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_5; - } - } - int t_6 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew_1052 + 128 + d_lane]; - float dkf_6 = pool_f32[ew_1052 + 1152 + d_lane]; - float dgf_6 = pool_f32[ew_1052 + 2176 + d_lane]; - float bt_t_6 = btv_1058[1]; - float e_off_6 = approx_exp2(g_t_6 - gn_1049); - float e_dg_6 = approx_exp2(g_t_6 - gc_1050); - float dq2_6 = __fmaf_rn(_tmem_load_12[1], e_off_6, _tmem_load_14[1] * e_dg_6); - float dk2_6 = __fmaf_rn(_tmem_load_13[1], e_off_6, _tmem_load_15[1] * e_dg_6); - dbp_1057[1] = dk2_6 * k_t_6; - float dk2b_6 = dk2_6 * bt_t_6; - float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[1]; - float dg_o_6 = __fmaf_rn(q_t_1_14, dq2_6, (dk2b_6 - _tmem_load_16[1]) * k_t_6) + dgf_6; - s_run_1059[0] = s_run_1059[0] + dg_o_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_6 = t_6 * qrs; - if (t_6 < clen) { - { - auto _gst_253 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_6; - } - { - auto _gst_254 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_6; - } - } - int t_7 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew_1052 + d_lane]; - float dkf_7 = pool_f32[ew_1052 + 1024 + d_lane]; - float dgf_7 = pool_f32[ew_1052 + 2048 + d_lane]; - float bt_t_7 = btv_1058[0]; - float e_off_7 = approx_exp2(g_t_7 - gn_1049); - float e_dg_7 = approx_exp2(g_t_7 - gc_1050); - float dq2_7 = __fmaf_rn(_tmem_load_12[0], e_off_7, _tmem_load_14[0] * e_dg_7); - float dk2_7 = __fmaf_rn(_tmem_load_13[0], e_off_7, _tmem_load_15[0] * e_dg_7); - dbp_1057[0] = dk2_7 * k_t_7; - float dk2b_7 = dk2_7 * bt_t_7; - float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[0]; - float dg_o_7 = __fmaf_rn(q_t_1_15, dq2_7, (dk2b_7 - _tmem_load_16[0]) * k_t_7) + dgf_7; - s_run_1059[0] = s_run_1059[0] + dg_o_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_7 = t_7 * qrs; - if (t_7 < clen) { - { - auto _gst_255 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_7; - } - { - auto _gst_256 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_7; + float r_251[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_0 = r_251[0]; + float gc_1 = gn_0 - 21.640425f; + int eb_0 = 98304 + th * -32768; + int ew_0 = eb_0 >> 2; + int qbase_1 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); + int sb8_0 = 48 + 8 * th; + int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); + int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); + int c_qa_0 = 160 + 8 * th; + int c_ka_0 = 176 + 8 * th; + int c_qd_0 = 224 + 8 * th; + int c_kd_0 = 240 + 8 * th; + int c_dk_0 = 48 + 8 * th; + int c_bp_0 = c_ka_0; + float s_run_0[1]; + s_run_0[0] = 0.0f; + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_1 = 7 - k_1; + float a_qd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1))); + float a_kd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_0 + j_1))); + float a_dk_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_0 + j_1))); + float a_qa_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa_0 + j_1))); + float a_ka_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka_0 + j_1))); + float btr_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_0 + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_0 + j_1; + float r_252[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_252[0]; + unsigned int r_253[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_253[0] << 16); + unsigned int r_254[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1 = __uint_as_float(r_254[0] << 16); + float q_m_1 = ((t_1 >= clen) ? 0.0f : q_t_1); + float dqf_1 = pool_f32[ew_0 + 128 * j_1 + d_lane]; + float dkf_1 = pool_f32[ew_0 + (1024 + 128 * j_1) + d_lane]; + float dgf_1 = pool_f32[ew_0 + (2048 + 128 * j_1) + d_lane]; + float bt_t_1 = btr_1[0]; + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float e_off_1 = approx_exp2(g_t_1 - gn_0); + float dq2_1 = __fmaf_rn(a_qa_1[0], e_off_1, a_qd_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(a_ka_1[0], e_off_1, a_kd_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + a_dk_1[0]; + float qe_1 = q_m_1 * e_dg_1; + float kp_12 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_12; + float kbe_1 = (float)kb_1 * e_dg_1; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, a_qd_1[0], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_12 * e_dg_1) * a_kd_1[0]); + float dg_o_1 = __fmaf_rn(q_m_1, dq2_1, __fmaf_rn(dk2b_1 - a_dk_1[0], k_t_1, cr_1)) + dgf_1; + s_run_0[0] = s_run_0[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_1)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff_1 = t_1 * qrs; + if (t_1 < clen) { + { + auto _gst_259 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_1; + } + { + auto _gst_260 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_1; + } } } - float tot_1228 = s_run_1059[0]; - int b4_1229 = lane_1 >> 4 & 1; - int b3_1230 = lane_1 >> 3 & 1; - int b2_1231 = lane_1 >> 2 & 1; - float v4_1232[4]; - float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_257; - float send_1 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_258; - float send_2 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_259; - float send_3 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_260; - float v2_1237[2]; - float send_4 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_261; - float send_5 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_262; - float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_265; + float tot_0 = s_run_0[0]; + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int b4_0 = lane_1 >> 4 & 1; + int b3_0 = lane_1 >> 3 & 1; + int b2_0 = lane_1 >> 2 & 1; + float v4_0[4]; + float send_4 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_261; + float send_5 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_5, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_262; + float send_6 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_263; + float send_7 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_264; + float v2_16[2]; + float send2_2 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send2_2, 8); + v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_265; + float send2_3 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send2_3, 8); + v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_266; + float send1_0 = ((b2_0 == 0) ? v2_16[1] : v2_16[0]); + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send1_0, 4); + float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_267; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_268; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_269; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); float gacc[8]; @@ -3803,8 +3388,8 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_1228) : "memory"); + uint32_t _addr_270 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_270), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3822,8 +3407,8 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); float t_other[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi = ((th == 1) ? tot_1228 : t_other[0]); - float t_lo = ((th == 1) ? t_other[0] : tot_1228); + float t_hi = ((th == 1) ? tot_0 : t_other[0]); + float t_lo = ((th == 1) ? t_other[0] : tot_0); float offset = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi); float sfx[8]; tmem_ld_x8(&sfx[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); @@ -3836,132 +3421,132 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_267 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_271 = make_float2(gacc[4], gacc[4]); float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&rp_u)); - float _abs_270; - asm("abs.f32 %0, %1;" : "=f"(_abs_270) : "f"(rp_z.x)); - float _neg_269; - asm("neg.f32 %0, %1;" : "=f"(_neg_269) : "f"(_abs_270)); - float _exp_268 = __nv_expf(_neg_269); - float _abs_273; - asm("abs.f32 %0, %1;" : "=f"(_abs_273) : "f"(rp_z.y)); - float _neg_272; - asm("neg.f32 %0, %1;" : "=f"(_neg_272) : "f"(_abs_273)); - float _exp_271 = __nv_expf(_neg_272); - float2 rp_e = make_float2(_exp_268, _exp_271); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_271), "l"(*(const unsigned long long*)&rp_u)); + float _abs_274; + asm("abs.f32 %0, %1;" : "=f"(_abs_274) : "f"(rp_z.x)); + float _neg_273; + asm("neg.f32 %0, %1;" : "=f"(_neg_273) : "f"(_abs_274)); + float _exp_272 = __nv_expf(_neg_273); + float _abs_277; + asm("abs.f32 %0, %1;" : "=f"(_abs_277) : "f"(rp_z.y)); + float _neg_276; + asm("neg.f32 %0, %1;" : "=f"(_neg_276) : "f"(_abs_277)); + float _exp_275 = __nv_expf(_neg_276); + float2 rp_e = make_float2(_exp_272, _exp_275); float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_274 = approx_rcp(rp_den.x); - float _rcp_275 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_276 = make_float2(_rcp_274, _rcp_275); + float _rcp_278 = approx_rcp(rp_den.x); + float _rcp_279 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_280 = make_float2(_rcp_278, _rcp_279); float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_276)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_280)); float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_277 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_281 = make_float2(gacc[5], gacc[5]); float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_281)); u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_278 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_282 = make_float2(gacc[4], gacc[4]); float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_278), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_281; - asm("abs.f32 %0, %1;" : "=f"(_abs_281) : "f"(rp_z_0.x)); - float _neg_280; - asm("neg.f32 %0, %1;" : "=f"(_neg_280) : "f"(_abs_281)); - float _exp_279 = __nv_expf(_neg_280); - float _abs_284; - asm("abs.f32 %0, %1;" : "=f"(_abs_284) : "f"(rp_z_0.y)); - float _neg_283; - asm("neg.f32 %0, %1;" : "=f"(_neg_283) : "f"(_abs_284)); - float _exp_282 = __nv_expf(_neg_283); - float2 rp_e_0 = make_float2(_exp_279, _exp_282); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_282), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_285; + asm("abs.f32 %0, %1;" : "=f"(_abs_285) : "f"(rp_z_0.x)); + float _neg_284; + asm("neg.f32 %0, %1;" : "=f"(_neg_284) : "f"(_abs_285)); + float _exp_283 = __nv_expf(_neg_284); + float _abs_288; + asm("abs.f32 %0, %1;" : "=f"(_abs_288) : "f"(rp_z_0.y)); + float _neg_287; + asm("neg.f32 %0, %1;" : "=f"(_neg_287) : "f"(_abs_288)); + float _exp_286 = __nv_expf(_neg_287); + float2 rp_e_0 = make_float2(_exp_283, _exp_286); float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_285 = approx_rcp(rp_den_0.x); - float _rcp_286 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_287 = make_float2(_rcp_285, _rcp_286); + float _rcp_289 = approx_rcp(rp_den_0.x); + float _rcp_290 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_291 = make_float2(_rcp_289, _rcp_290); float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_287)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_291)); float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_288 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_292 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_288)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_292)); u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_289 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_293 = make_float2(gacc[4], gacc[4]); float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_289), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_292; - asm("abs.f32 %0, %1;" : "=f"(_abs_292) : "f"(rp_z_1.x)); - float _neg_291; - asm("neg.f32 %0, %1;" : "=f"(_neg_291) : "f"(_abs_292)); - float _exp_290 = __nv_expf(_neg_291); - float _abs_295; - asm("abs.f32 %0, %1;" : "=f"(_abs_295) : "f"(rp_z_1.y)); - float _neg_294; - asm("neg.f32 %0, %1;" : "=f"(_neg_294) : "f"(_abs_295)); - float _exp_293 = __nv_expf(_neg_294); - float2 rp_e_1 = make_float2(_exp_290, _exp_293); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_293), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_296; + asm("abs.f32 %0, %1;" : "=f"(_abs_296) : "f"(rp_z_1.x)); + float _neg_295; + asm("neg.f32 %0, %1;" : "=f"(_neg_295) : "f"(_abs_296)); + float _exp_294 = __nv_expf(_neg_295); + float _abs_299; + asm("abs.f32 %0, %1;" : "=f"(_abs_299) : "f"(rp_z_1.y)); + float _neg_298; + asm("neg.f32 %0, %1;" : "=f"(_neg_298) : "f"(_abs_299)); + float _exp_297 = __nv_expf(_neg_298); + float2 rp_e_1 = make_float2(_exp_294, _exp_297); float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_296 = approx_rcp(rp_den_1.x); - float _rcp_297 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_298 = make_float2(_rcp_296, _rcp_297); + float _rcp_300 = approx_rcp(rp_den_1.x); + float _rcp_301 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_302 = make_float2(_rcp_300, _rcp_301); float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_298)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_302)); float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_299 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_303 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_299)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_303)); u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_300 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_304 = make_float2(gacc[4], gacc[4]); float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_303; - asm("abs.f32 %0, %1;" : "=f"(_abs_303) : "f"(rp_z_2.x)); - float _neg_302; - asm("neg.f32 %0, %1;" : "=f"(_neg_302) : "f"(_abs_303)); - float _exp_301 = __nv_expf(_neg_302); - float _abs_306; - asm("abs.f32 %0, %1;" : "=f"(_abs_306) : "f"(rp_z_2.y)); - float _neg_305; - asm("neg.f32 %0, %1;" : "=f"(_neg_305) : "f"(_abs_306)); - float _exp_304 = __nv_expf(_neg_305); - float2 rp_e_2 = make_float2(_exp_301, _exp_304); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_304), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_307; + asm("abs.f32 %0, %1;" : "=f"(_abs_307) : "f"(rp_z_2.x)); + float _neg_306; + asm("neg.f32 %0, %1;" : "=f"(_neg_306) : "f"(_abs_307)); + float _exp_305 = __nv_expf(_neg_306); + float _abs_310; + asm("abs.f32 %0, %1;" : "=f"(_abs_310) : "f"(rp_z_2.y)); + float _neg_309; + asm("neg.f32 %0, %1;" : "=f"(_neg_309) : "f"(_abs_310)); + float _exp_308 = __nv_expf(_neg_309); + float2 rp_e_2 = make_float2(_exp_305, _exp_308); float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_307 = approx_rcp(rp_den_2.x); - float _rcp_308 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_309 = make_float2(_rcp_307, _rcp_308); + float _rcp_311 = approx_rcp(rp_den_2.x); + float _rcp_312 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_313 = make_float2(_rcp_311, _rcp_312); float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_313)); float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_310 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_314 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_310)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_314)); u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -3976,54 +3561,54 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_311; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_311) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_311; + uint32_t _prmt_315; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_315; int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_312 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = pq_0[0]; + auto _gst_316 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = pq_0[0]; } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_313; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_313) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_313; + uint32_t _prmt_317; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_317; int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_314 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = pq_1[0]; + auto _gst_318 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = pq_1[0]; } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_315; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_315; + uint32_t _prmt_319; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_319) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_319; int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_316 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = pq_2[0]; + auto _gst_320 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = pq_2[0]; } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_317; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_317; + uint32_t _prmt_321; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_321) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_321; int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_318 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = pq_3[0]; + auto _gst_322 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast(_gst_322) + (0)) = pq_3[0]; } } int t_a_0 = 48 + 8 * th; @@ -4088,47 +3673,47 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_319; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_319) : "f"(s_a), "f"(nm_t)); + float _sub_323; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_323) : "f"(s_a), "f"(nm_t)); float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_319), "f"(pa)); - float _sub_320; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_320) : "f"(pa), "f"(nm_t)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_323), "f"(pa)); + float _sub_324; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_324) : "f"(pa), "f"(nm_t)); float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_320), "f"(s_a)); - float _abs_322; - asm("abs.f32 %0, %1;" : "=f"(_abs_322) : "f"(s_a)); - float _abs_323; - asm("abs.f32 %0, %1;" : "=f"(_abs_323) : "f"(pa)); - uint32_t _setp_321_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_321_bits) : "f"(_abs_322), "f"(_abs_323)); - bool _setp_321 = _setp_321_bits != 0; - float nm_err = ((_setp_321) ? nm_es : nm_ex); - float _add_324; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(c_a), "f"(nm_err)); - c_a = _add_324; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_324), "f"(s_a)); + float _abs_326; + asm("abs.f32 %0, %1;" : "=f"(_abs_326) : "f"(s_a)); + float _abs_327; + asm("abs.f32 %0, %1;" : "=f"(_abs_327) : "f"(pa)); + uint32_t _setp_325_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_325_bits) : "f"(_abs_326), "f"(_abs_327)); + bool _setp_325 = _setp_325_bits != 0; + float nm_err = ((_setp_325) ? nm_es : nm_ex); + float _add_328; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(c_a), "f"(nm_err)); + c_a = _add_328; s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_325; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_325) : "f"(s_b), "f"(nm_t_0)); + float _sub_329; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_329) : "f"(s_b), "f"(nm_t_0)); float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_325), "f"(pb)); - float _sub_326; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_326) : "f"(pb), "f"(nm_t_0)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_329), "f"(pb)); + float _sub_330; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_330) : "f"(pb), "f"(nm_t_0)); float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_326), "f"(s_b)); - float _abs_328; - asm("abs.f32 %0, %1;" : "=f"(_abs_328) : "f"(s_b)); - float _abs_329; - asm("abs.f32 %0, %1;" : "=f"(_abs_329) : "f"(pb)); - uint32_t _setp_327_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_327_bits) : "f"(_abs_328), "f"(_abs_329)); - bool _setp_327 = _setp_327_bits != 0; - float nm_err_0 = ((_setp_327) ? nm_es_0 : nm_ex_0); - float _add_330; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_330; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_330), "f"(s_b)); + float _abs_332; + asm("abs.f32 %0, %1;" : "=f"(_abs_332) : "f"(s_b)); + float _abs_333; + asm("abs.f32 %0, %1;" : "=f"(_abs_333) : "f"(pb)); + uint32_t _setp_331_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_331_bits) : "f"(_abs_332), "f"(_abs_333)); + bool _setp_331 = _setp_331_bits != 0; + float nm_err_0 = ((_setp_331) ? nm_es_0 : nm_ex_0); + float _add_334; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_334; s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4152,132 +3737,132 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_331 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_335 = make_float2(gacc[4], gacc[4]); float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_331), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_334; - asm("abs.f32 %0, %1;" : "=f"(_abs_334) : "f"(rp_z_3.x)); - float _neg_333; - asm("neg.f32 %0, %1;" : "=f"(_neg_333) : "f"(_abs_334)); - float _exp_332 = __nv_expf(_neg_333); - float _abs_337; - asm("abs.f32 %0, %1;" : "=f"(_abs_337) : "f"(rp_z_3.y)); - float _neg_336; - asm("neg.f32 %0, %1;" : "=f"(_neg_336) : "f"(_abs_337)); - float _exp_335 = __nv_expf(_neg_336); - float2 rp_e_3 = make_float2(_exp_332, _exp_335); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_335), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_338; + asm("abs.f32 %0, %1;" : "=f"(_abs_338) : "f"(rp_z_3.x)); + float _neg_337; + asm("neg.f32 %0, %1;" : "=f"(_neg_337) : "f"(_abs_338)); + float _exp_336 = __nv_expf(_neg_337); + float _abs_341; + asm("abs.f32 %0, %1;" : "=f"(_abs_341) : "f"(rp_z_3.y)); + float _neg_340; + asm("neg.f32 %0, %1;" : "=f"(_neg_340) : "f"(_abs_341)); + float _exp_339 = __nv_expf(_neg_340); + float2 rp_e_3 = make_float2(_exp_336, _exp_339); float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_338 = approx_rcp(rp_den_3.x); - float _rcp_339 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_340 = make_float2(_rcp_338, _rcp_339); + float _rcp_342 = approx_rcp(rp_den_3.x); + float _rcp_343 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_344 = make_float2(_rcp_342, _rcp_343); float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_340)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_344)); float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_341 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_345 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_345)); u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_342 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_346 = make_float2(gacc[4], gacc[4]); float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_342), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_345; - asm("abs.f32 %0, %1;" : "=f"(_abs_345) : "f"(rp_z_4.x)); - float _neg_344; - asm("neg.f32 %0, %1;" : "=f"(_neg_344) : "f"(_abs_345)); - float _exp_343 = __nv_expf(_neg_344); - float _abs_348; - asm("abs.f32 %0, %1;" : "=f"(_abs_348) : "f"(rp_z_4.y)); - float _neg_347; - asm("neg.f32 %0, %1;" : "=f"(_neg_347) : "f"(_abs_348)); - float _exp_346 = __nv_expf(_neg_347); - float2 rp_e_4 = make_float2(_exp_343, _exp_346); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_346), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_349; + asm("abs.f32 %0, %1;" : "=f"(_abs_349) : "f"(rp_z_4.x)); + float _neg_348; + asm("neg.f32 %0, %1;" : "=f"(_neg_348) : "f"(_abs_349)); + float _exp_347 = __nv_expf(_neg_348); + float _abs_352; + asm("abs.f32 %0, %1;" : "=f"(_abs_352) : "f"(rp_z_4.y)); + float _neg_351; + asm("neg.f32 %0, %1;" : "=f"(_neg_351) : "f"(_abs_352)); + float _exp_350 = __nv_expf(_neg_351); + float2 rp_e_4 = make_float2(_exp_347, _exp_350); float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_349 = approx_rcp(rp_den_4.x); - float _rcp_350 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_351 = make_float2(_rcp_349, _rcp_350); + float _rcp_353 = approx_rcp(rp_den_4.x); + float _rcp_354 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_355 = make_float2(_rcp_353, _rcp_354); float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_351)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_355)); float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_352 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_356 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_352)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_356)); u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_353 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_357 = make_float2(gacc[4], gacc[4]); float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_356; - asm("abs.f32 %0, %1;" : "=f"(_abs_356) : "f"(rp_z_5.x)); - float _neg_355; - asm("neg.f32 %0, %1;" : "=f"(_neg_355) : "f"(_abs_356)); - float _exp_354 = __nv_expf(_neg_355); - float _abs_359; - asm("abs.f32 %0, %1;" : "=f"(_abs_359) : "f"(rp_z_5.y)); - float _neg_358; - asm("neg.f32 %0, %1;" : "=f"(_neg_358) : "f"(_abs_359)); - float _exp_357 = __nv_expf(_neg_358); - float2 rp_e_5 = make_float2(_exp_354, _exp_357); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_357), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_360; + asm("abs.f32 %0, %1;" : "=f"(_abs_360) : "f"(rp_z_5.x)); + float _neg_359; + asm("neg.f32 %0, %1;" : "=f"(_neg_359) : "f"(_abs_360)); + float _exp_358 = __nv_expf(_neg_359); + float _abs_363; + asm("abs.f32 %0, %1;" : "=f"(_abs_363) : "f"(rp_z_5.y)); + float _neg_362; + asm("neg.f32 %0, %1;" : "=f"(_neg_362) : "f"(_abs_363)); + float _exp_361 = __nv_expf(_neg_362); + float2 rp_e_5 = make_float2(_exp_358, _exp_361); float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_360 = approx_rcp(rp_den_5.x); - float _rcp_361 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_362 = make_float2(_rcp_360, _rcp_361); + float _rcp_364 = approx_rcp(rp_den_5.x); + float _rcp_365 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_366 = make_float2(_rcp_364, _rcp_365); float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_362)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_366)); float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_363 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_367 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_363)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_367)); u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_364 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_368 = make_float2(gacc[4], gacc[4]); float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_364), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_367; - asm("abs.f32 %0, %1;" : "=f"(_abs_367) : "f"(rp_z_6.x)); - float _neg_366; - asm("neg.f32 %0, %1;" : "=f"(_neg_366) : "f"(_abs_367)); - float _exp_365 = __nv_expf(_neg_366); - float _abs_370; - asm("abs.f32 %0, %1;" : "=f"(_abs_370) : "f"(rp_z_6.y)); - float _neg_369; - asm("neg.f32 %0, %1;" : "=f"(_neg_369) : "f"(_abs_370)); - float _exp_368 = __nv_expf(_neg_369); - float2 rp_e_6 = make_float2(_exp_365, _exp_368); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_368), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_371; + asm("abs.f32 %0, %1;" : "=f"(_abs_371) : "f"(rp_z_6.x)); + float _neg_370; + asm("neg.f32 %0, %1;" : "=f"(_neg_370) : "f"(_abs_371)); + float _exp_369 = __nv_expf(_neg_370); + float _abs_374; + asm("abs.f32 %0, %1;" : "=f"(_abs_374) : "f"(rp_z_6.y)); + float _neg_373; + asm("neg.f32 %0, %1;" : "=f"(_neg_373) : "f"(_abs_374)); + float _exp_372 = __nv_expf(_neg_373); + float2 rp_e_6 = make_float2(_exp_369, _exp_372); float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_371 = approx_rcp(rp_den_6.x); - float _rcp_372 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_373 = make_float2(_rcp_371, _rcp_372); + float _rcp_375 = approx_rcp(rp_den_6.x); + float _rcp_376 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_377 = make_float2(_rcp_375, _rcp_376); float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_373)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_377)); float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_374 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_378 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_374)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_378)); u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4292,54 +3877,54 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_375; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_375) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_375; + uint32_t _prmt_379; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_379; int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_376 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_376)); - *(reinterpret_cast(_gst_376) + (0)) = pq_4[0]; + auto _gst_380 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = pq_4[0]; } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_377; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_377) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_377; + uint32_t _prmt_381; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_381; int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_378 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_378)); - *(reinterpret_cast(_gst_378) + (0)) = pq_5[0]; + auto _gst_382 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = pq_5[0]; } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_379; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_379; + uint32_t _prmt_383; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_383) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_383; int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_380 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_380)); - *(reinterpret_cast(_gst_380) + (0)) = pq_6[0]; + auto _gst_384 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = pq_6[0]; } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_381; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_381; + uint32_t _prmt_385; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_385) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_385; int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_382 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_382)); - *(reinterpret_cast(_gst_382) + (0)) = pq_7[0]; + auto _gst_386 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_386)); + *(reinterpret_cast(_gst_386) + (0)) = pq_7[0]; } } int t_a_8 = 32 + 8 * th; @@ -4404,47 +3989,47 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_383; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_383) : "f"(s_a_0), "f"(nm_t_1)); + float _sub_387; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_387) : "f"(s_a_0), "f"(nm_t_1)); float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_383), "f"(pa_0)); - float _sub_384; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_384) : "f"(pa_0), "f"(nm_t_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_387), "f"(pa_0)); + float _sub_388; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_388) : "f"(pa_0), "f"(nm_t_1)); float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_384), "f"(s_a_0)); - float _abs_386; - asm("abs.f32 %0, %1;" : "=f"(_abs_386) : "f"(s_a_0)); - float _abs_387; - asm("abs.f32 %0, %1;" : "=f"(_abs_387) : "f"(pa_0)); - uint32_t _setp_385_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_385_bits) : "f"(_abs_386), "f"(_abs_387)); - bool _setp_385 = _setp_385_bits != 0; - float nm_err_1 = ((_setp_385) ? nm_es_1 : nm_ex_1); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_388), "f"(s_a_0)); + float _abs_390; + asm("abs.f32 %0, %1;" : "=f"(_abs_390) : "f"(s_a_0)); + float _abs_391; + asm("abs.f32 %0, %1;" : "=f"(_abs_391) : "f"(pa_0)); + uint32_t _setp_389_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_389_bits) : "f"(_abs_390), "f"(_abs_391)); + bool _setp_389 = _setp_389_bits != 0; + float nm_err_1 = ((_setp_389) ? nm_es_1 : nm_ex_1); + float _add_392; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_392; s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_389; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_389) : "f"(s_b_0), "f"(nm_t_2)); + float _sub_393; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_393) : "f"(s_b_0), "f"(nm_t_2)); float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_389), "f"(pb_0)); - float _sub_390; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_390) : "f"(pb_0), "f"(nm_t_2)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_393), "f"(pb_0)); + float _sub_394; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_394) : "f"(pb_0), "f"(nm_t_2)); float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_390), "f"(s_b_0)); - float _abs_392; - asm("abs.f32 %0, %1;" : "=f"(_abs_392) : "f"(s_b_0)); - float _abs_393; - asm("abs.f32 %0, %1;" : "=f"(_abs_393) : "f"(pb_0)); - uint32_t _setp_391_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_391_bits) : "f"(_abs_392), "f"(_abs_393)); - bool _setp_391 = _setp_391_bits != 0; - float nm_err_2 = ((_setp_391) ? nm_es_2 : nm_ex_2); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_394; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_394), "f"(s_b_0)); + float _abs_396; + asm("abs.f32 %0, %1;" : "=f"(_abs_396) : "f"(s_b_0)); + float _abs_397; + asm("abs.f32 %0, %1;" : "=f"(_abs_397) : "f"(pb_0)); + uint32_t _setp_395_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_395_bits) : "f"(_abs_396), "f"(_abs_397)); + bool _setp_395 = _setp_395_bits != 0; + float nm_err_2 = ((_setp_395) ? nm_es_2 : nm_ex_2); + float _add_398; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_398; s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4452,8 +4037,8 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_395 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_395), "f"(off_acc[0]) : "memory"); + uint32_t _addr_399 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_399), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4479,9 +4064,9 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1292 = r_297[0]; + float r_255[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1292 = r_255[0]; float gc_1293 = gn_1292 - 21.640425f; int eb_1294 = 81920 + th * -32768; int ew_1295 = eb_1294 >> 2; @@ -4510,348 +4095,388 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1302[1]; s_run_1302[0] = 0.0f; - int t_8 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_1295 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_1295 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_1295 + 2944 + d_lane]; - float bt_t_8 = btv_1301[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_1292); - float e_dg_8 = approx_exp2(g_t_8 - gc_1293); - float dq2_8 = __fmaf_rn(_tmem_load_20[7], e_off_8, _tmem_load_22[7] * e_dg_8); - float dk2_8 = __fmaf_rn(_tmem_load_21[7], e_off_8, _tmem_load_23[7] * e_dg_8); - dbp_1300[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_24[7]; - float dg_o_8 = __fmaf_rn(q_t_1_16, dq2_8, (dk2b_8 - _tmem_load_24[7]) * k_t_8) + dgf_8; - s_run_1302[0] = s_run_1302[0] + dg_o_8; + int t_0 = 16 + 8 * th + 7; + float r_256[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_0 = r_256[0]; + unsigned int r_257[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_0 = __uint_as_float(r_257[0] << 16); + unsigned int r_258[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_0 = __uint_as_float(r_258[0] << 16); + float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); + float dqf_0 = pool_f32[ew_1295 + 896 + d_lane]; + float dkf_0 = pool_f32[ew_1295 + 1920 + d_lane]; + float dgf_0 = pool_f32[ew_1295 + 2944 + d_lane]; + float bt_t_0 = btv_1301[7]; + float e_off_0 = approx_exp2(g_t_0 - gn_1292); + float e_dg_0 = approx_exp2(g_t_0 - gc_1293); + float dq2_0 = __fmaf_rn(_tmem_load_20[7], e_off_0, _tmem_load_22[7] * e_dg_0); + float dk2_0 = __fmaf_rn(_tmem_load_21[7], e_off_0, _tmem_load_23[7] * e_dg_0); + dbp_1300[7] = dk2_0 * k_t_0; + float dk2b_0 = dk2_0 * bt_t_0; + float dq_o_0 = dq2_0 + dqf_0; + float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_24[7]; + float qe_2 = q_t_1_0 * e_dg_0; + float kp_13 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_13; + float kbe_2 = (float)kb_2 * e_dg_0; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_13 * e_dg_0) * _tmem_load_23[7]); + float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_24[7], k_t_0, cr_2)) + dgf_0; + s_run_1302[0] = s_run_1302[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_8 = t_8 * qrs; - if (t_8 < clen) { + int qoff_0 = t_0 * qrs; + if (t_0 < clen) { { - auto _gst_396 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_396)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_396) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_400 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_400)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_397 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_397)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_397) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_401 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_401)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_0; } } - int t_9 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_1295 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_1295 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_1295 + 2816 + d_lane]; - float bt_t_9 = btv_1301[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_1292); - float e_dg_9 = approx_exp2(g_t_9 - gc_1293); - float dq2_9 = __fmaf_rn(_tmem_load_20[6], e_off_9, _tmem_load_22[6] * e_dg_9); - float dk2_9 = __fmaf_rn(_tmem_load_21[6], e_off_9, _tmem_load_23[6] * e_dg_9); - dbp_1300[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[6]; - float dg_o_9 = __fmaf_rn(q_t_1_17, dq2_9, (dk2b_9 - _tmem_load_24[6]) * k_t_9) + dgf_9; - s_run_1302[0] = s_run_1302[0] + dg_o_9; + int t_1_1 = 16 + 8 * th + 6; + float r_259[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1_1 = r_259[0]; + unsigned int r_260[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1_1 = __uint_as_float(r_260[0] << 16); + unsigned int r_261[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1_1 = __uint_as_float(r_261[0] << 16); + float q_t_1_1_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_1); + float dqf_1_1 = pool_f32[ew_1295 + 768 + d_lane]; + float dkf_1_1 = pool_f32[ew_1295 + 1792 + d_lane]; + float dgf_1_1 = pool_f32[ew_1295 + 2816 + d_lane]; + float bt_t_1_1 = btv_1301[6]; + float e_off_1_1 = approx_exp2(g_t_1_1 - gn_1292); + float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_1293); + float dq2_1_1 = __fmaf_rn(_tmem_load_20[6], e_off_1_1, _tmem_load_22[6] * e_dg_1_1); + float dk2_1_1 = __fmaf_rn(_tmem_load_21[6], e_off_1_1, _tmem_load_23[6] * e_dg_1_1); + dbp_1300[6] = dk2_1_1 * k_t_1_1; + float dk2b_1_1 = dk2_1_1 * bt_t_1_1; + float dq_o_1_1 = dq2_1_1 + dqf_1_1; + float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_24[6]; + float qe_0 = q_t_1_1_1 * e_dg_1_1; + float kp_12_1 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12_1; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12_1 * e_dg_1_1) * _tmem_load_23[6]); + float dg_o_1_1 = __fmaf_rn(q_t_1_1_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_24[6], k_t_1_1, cr_0)) + dgf_1_1; + s_run_1302[0] = s_run_1302[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_9 = t_9 * qrs; - if (t_9 < clen) { + int qoff_1_1 = t_1_1 * qrs; + if (t_1_1 < clen) { { - auto _gst_398 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_398)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_398) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_402 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_402)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_1_1; } { - auto _gst_399 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_399)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_399) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_403 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_403)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_1_1; } } - int t_10 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_1295 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_1295 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_1295 + 2688 + d_lane]; - float bt_t_10 = btv_1301[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_1292); - float e_dg_10 = approx_exp2(g_t_10 - gc_1293); - float dq2_10 = __fmaf_rn(_tmem_load_20[5], e_off_10, _tmem_load_22[5] * e_dg_10); - float dk2_10 = __fmaf_rn(_tmem_load_21[5], e_off_10, _tmem_load_23[5] * e_dg_10); - dbp_1300[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[5]; - float dg_o_10 = __fmaf_rn(q_t_1_18, dq2_10, (dk2b_10 - _tmem_load_24[5]) * k_t_10) + dgf_10; - s_run_1302[0] = s_run_1302[0] + dg_o_10; + int t_2 = 16 + 8 * th + 5; + float r_262[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_262[0]; + unsigned int r_263[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_263[0] << 16); + unsigned int r_264[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_264[0] << 16); + float q_t_1_2 = ((t_2 >= clen) ? 0.0f : q_t_2); + float dqf_2 = pool_f32[ew_1295 + 640 + d_lane]; + float dkf_2 = pool_f32[ew_1295 + 1664 + d_lane]; + float dgf_2 = pool_f32[ew_1295 + 2688 + d_lane]; + float bt_t_2 = btv_1301[5]; + float e_off_2 = approx_exp2(g_t_2 - gn_1292); + float e_dg_2 = approx_exp2(g_t_2 - gc_1293); + float dq2_2 = __fmaf_rn(_tmem_load_20[5], e_off_2, _tmem_load_22[5] * e_dg_2); + float dk2_2 = __fmaf_rn(_tmem_load_21[5], e_off_2, _tmem_load_23[5] * e_dg_2); + dbp_1300[5] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_24[5]; + float qe_1_1 = q_t_1_2 * e_dg_2; + float kp_13_1 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1_1 = (__nv_bfloat16)kp_13_1; + float kbe_1_1 = (float)kb_1_1 * e_dg_2; + float cr_1_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1_1) & 4294959104u) - qe_1_1, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_1_1) & 4294959104u) - kp_13_1 * e_dg_2) * _tmem_load_23[5]); + float dg_o_2 = __fmaf_rn(q_t_1_2, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_24[5], k_t_2, cr_1_1)) + dgf_2; + s_run_1302[0] = s_run_1302[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_10 = t_10 * qrs; - if (t_10 < clen) { + int qoff_2 = t_2 * qrs; + if (t_2 < clen) { { - auto _gst_400 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_400)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_404 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_404)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_401 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_401)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_405 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_405)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_2; } } - int t_11 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_1295 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_1295 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_1295 + 2560 + d_lane]; - float bt_t_11 = btv_1301[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_1292); - float e_dg_11 = approx_exp2(g_t_11 - gc_1293); - float dq2_11 = __fmaf_rn(_tmem_load_20[4], e_off_11, _tmem_load_22[4] * e_dg_11); - float dk2_11 = __fmaf_rn(_tmem_load_21[4], e_off_11, _tmem_load_23[4] * e_dg_11); - dbp_1300[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[4]; - float dg_o_11 = __fmaf_rn(q_t_1_19, dq2_11, (dk2b_11 - _tmem_load_24[4]) * k_t_11) + dgf_11; - s_run_1302[0] = s_run_1302[0] + dg_o_11; + int t_3 = 16 + 8 * th + 4; + float r_265[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_3 = r_265[0]; + unsigned int r_266[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_3 = __uint_as_float(r_266[0] << 16); + unsigned int r_267[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_3 = __uint_as_float(r_267[0] << 16); + float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); + float dqf_3 = pool_f32[ew_1295 + 512 + d_lane]; + float dkf_3 = pool_f32[ew_1295 + 1536 + d_lane]; + float dgf_3 = pool_f32[ew_1295 + 2560 + d_lane]; + float bt_t_3 = btv_1301[4]; + float e_off_3 = approx_exp2(g_t_3 - gn_1292); + float e_dg_3 = approx_exp2(g_t_3 - gc_1293); + float dq2_3 = __fmaf_rn(_tmem_load_20[4], e_off_3, _tmem_load_22[4] * e_dg_3); + float dk2_3 = __fmaf_rn(_tmem_load_21[4], e_off_3, _tmem_load_23[4] * e_dg_3); + dbp_1300[4] = dk2_3 * k_t_3; + float dk2b_3 = dk2_3 * bt_t_3; + float dq_o_3 = dq2_3 + dqf_3; + float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_24[4]; + float qe_2_1 = q_t_1_3 * e_dg_3; + float kp_14 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2_1 = (__nv_bfloat16)kp_14; + float kbe_2_1 = (float)kb_2_1 * e_dg_3; + float cr_2_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_2_1) & 4294959104u) - qe_2_1, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_2_1) & 4294959104u) - kp_14 * e_dg_3) * _tmem_load_23[4]); + float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_24[4], k_t_3, cr_2_1)) + dgf_3; + s_run_1302[0] = s_run_1302[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_11 = t_11 * qrs; - if (t_11 < clen) { + int qoff_3 = t_3 * qrs; + if (t_3 < clen) { { - auto _gst_402 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_402)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_406 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_406)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_403 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_403)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_407 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_407)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_3; } } - int t_12 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_1295 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_1295 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_1295 + 2432 + d_lane]; - float bt_t_12 = btv_1301[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_1292); - float e_dg_12 = approx_exp2(g_t_12 - gc_1293); - float dq2_12 = __fmaf_rn(_tmem_load_20[3], e_off_12, _tmem_load_22[3] * e_dg_12); - float dk2_12 = __fmaf_rn(_tmem_load_21[3], e_off_12, _tmem_load_23[3] * e_dg_12); - dbp_1300[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[3]; - float dg_o_12 = __fmaf_rn(q_t_1_20, dq2_12, (dk2b_12 - _tmem_load_24[3]) * k_t_12) + dgf_12; - s_run_1302[0] = s_run_1302[0] + dg_o_12; + int t_4 = 16 + 8 * th + 3; + float r_268[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_4 = r_268[0]; + unsigned int r_269[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_4 = __uint_as_float(r_269[0] << 16); + unsigned int r_270[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_4 = __uint_as_float(r_270[0] << 16); + float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); + float dqf_4 = pool_f32[ew_1295 + 384 + d_lane]; + float dkf_4 = pool_f32[ew_1295 + 1408 + d_lane]; + float dgf_4 = pool_f32[ew_1295 + 2432 + d_lane]; + float bt_t_4 = btv_1301[3]; + float e_off_4 = approx_exp2(g_t_4 - gn_1292); + float e_dg_4 = approx_exp2(g_t_4 - gc_1293); + float dq2_4 = __fmaf_rn(_tmem_load_20[3], e_off_4, _tmem_load_22[3] * e_dg_4); + float dk2_4 = __fmaf_rn(_tmem_load_21[3], e_off_4, _tmem_load_23[3] * e_dg_4); + dbp_1300[3] = dk2_4 * k_t_4; + float dk2b_4 = dk2_4 * bt_t_4; + float dq_o_4 = dq2_4 + dqf_4; + float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_24[3]; + float qe_3 = q_t_1_4 * e_dg_4; + float kp_15 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_4) * _tmem_load_23[3]); + float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_24[3], k_t_4, cr_3)) + dgf_4; + s_run_1302[0] = s_run_1302[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_12 = t_12 * qrs; - if (t_12 < clen) { + int qoff_4 = t_4 * qrs; + if (t_4 < clen) { { - auto _gst_404 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_404)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_408 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_408)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_405 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_405)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_409 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_409)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_4; } } - int t_13 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_1295 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_1295 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_1295 + 2304 + d_lane]; - float bt_t_13 = btv_1301[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_1292); - float e_dg_13 = approx_exp2(g_t_13 - gc_1293); - float dq2_13 = __fmaf_rn(_tmem_load_20[2], e_off_13, _tmem_load_22[2] * e_dg_13); - float dk2_13 = __fmaf_rn(_tmem_load_21[2], e_off_13, _tmem_load_23[2] * e_dg_13); - dbp_1300[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[2]; - float dg_o_13 = __fmaf_rn(q_t_1_21, dq2_13, (dk2b_13 - _tmem_load_24[2]) * k_t_13) + dgf_13; - s_run_1302[0] = s_run_1302[0] + dg_o_13; + int t_5 = 16 + 8 * th + 2; + float r_271[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_5 = r_271[0]; + unsigned int r_272[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_5 = __uint_as_float(r_272[0] << 16); + unsigned int r_273[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_5 = __uint_as_float(r_273[0] << 16); + float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); + float dqf_5 = pool_f32[ew_1295 + 256 + d_lane]; + float dkf_5 = pool_f32[ew_1295 + 1280 + d_lane]; + float dgf_5 = pool_f32[ew_1295 + 2304 + d_lane]; + float bt_t_5 = btv_1301[2]; + float e_off_5 = approx_exp2(g_t_5 - gn_1292); + float e_dg_5 = approx_exp2(g_t_5 - gc_1293); + float dq2_5 = __fmaf_rn(_tmem_load_20[2], e_off_5, _tmem_load_22[2] * e_dg_5); + float dk2_5 = __fmaf_rn(_tmem_load_21[2], e_off_5, _tmem_load_23[2] * e_dg_5); + dbp_1300[2] = dk2_5 * k_t_5; + float dk2b_5 = dk2_5 * bt_t_5; + float dq_o_5 = dq2_5 + dqf_5; + float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_24[2]; + float qe_4 = q_t_1_5 * e_dg_5; + float kp_16 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_5) * _tmem_load_23[2]); + float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_24[2], k_t_5, cr_4)) + dgf_5; + s_run_1302[0] = s_run_1302[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_13 = t_13 * qrs; - if (t_13 < clen) { + int qoff_5 = t_5 * qrs; + if (t_5 < clen) { { - auto _gst_406 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_406)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_410 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_410)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_407 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_407)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_411 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_411)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_5; } } - int t_14 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_1295 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_1295 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_1295 + 2176 + d_lane]; - float bt_t_14 = btv_1301[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_1292); - float e_dg_14 = approx_exp2(g_t_14 - gc_1293); - float dq2_14 = __fmaf_rn(_tmem_load_20[1], e_off_14, _tmem_load_22[1] * e_dg_14); - float dk2_14 = __fmaf_rn(_tmem_load_21[1], e_off_14, _tmem_load_23[1] * e_dg_14); - dbp_1300[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[1]; - float dg_o_14 = __fmaf_rn(q_t_1_22, dq2_14, (dk2b_14 - _tmem_load_24[1]) * k_t_14) + dgf_14; - s_run_1302[0] = s_run_1302[0] + dg_o_14; + int t_6 = 16 + 8 * th + 1; + float r_274[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_6 = r_274[0]; + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_6 = __uint_as_float(r_275[0] << 16); + unsigned int r_276[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_6 = __uint_as_float(r_276[0] << 16); + float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); + float dqf_6 = pool_f32[ew_1295 + 128 + d_lane]; + float dkf_6 = pool_f32[ew_1295 + 1152 + d_lane]; + float dgf_6 = pool_f32[ew_1295 + 2176 + d_lane]; + float bt_t_6 = btv_1301[1]; + float e_off_6 = approx_exp2(g_t_6 - gn_1292); + float e_dg_6 = approx_exp2(g_t_6 - gc_1293); + float dq2_6 = __fmaf_rn(_tmem_load_20[1], e_off_6, _tmem_load_22[1] * e_dg_6); + float dk2_6 = __fmaf_rn(_tmem_load_21[1], e_off_6, _tmem_load_23[1] * e_dg_6); + dbp_1300[1] = dk2_6 * k_t_6; + float dk2b_6 = dk2_6 * bt_t_6; + float dq_o_6 = dq2_6 + dqf_6; + float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_24[1]; + float qe_5 = q_t_1_6 * e_dg_6; + float kp_17 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_6) * _tmem_load_23[1]); + float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_24[1], k_t_6, cr_5)) + dgf_6; + s_run_1302[0] = s_run_1302[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_14 = t_14 * qrs; - if (t_14 < clen) { + int qoff_6 = t_6 * qrs; + if (t_6 < clen) { { - auto _gst_408 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_408)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_412 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_412)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_412) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_409 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_409)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_413 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_413)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_413) + (0)) = (__nv_bfloat16)dk_o_6; } } - int t_15 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_1295 + d_lane]; - float dkf_15 = pool_f32[ew_1295 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_1295 + 2048 + d_lane]; - float bt_t_15 = btv_1301[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_1292); - float e_dg_15 = approx_exp2(g_t_15 - gc_1293); - float dq2_15 = __fmaf_rn(_tmem_load_20[0], e_off_15, _tmem_load_22[0] * e_dg_15); - float dk2_15 = __fmaf_rn(_tmem_load_21[0], e_off_15, _tmem_load_23[0] * e_dg_15); - dbp_1300[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[0]; - float dg_o_15 = __fmaf_rn(q_t_1_23, dq2_15, (dk2b_15 - _tmem_load_24[0]) * k_t_15) + dgf_15; - s_run_1302[0] = s_run_1302[0] + dg_o_15; + int t_7 = 16 + 8 * th; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_7 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_7 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_7 = __uint_as_float(r_279[0] << 16); + float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); + float dqf_7 = pool_f32[ew_1295 + d_lane]; + float dkf_7 = pool_f32[ew_1295 + 1024 + d_lane]; + float dgf_7 = pool_f32[ew_1295 + 2048 + d_lane]; + float bt_t_7 = btv_1301[0]; + float e_off_7 = approx_exp2(g_t_7 - gn_1292); + float e_dg_7 = approx_exp2(g_t_7 - gc_1293); + float dq2_7 = __fmaf_rn(_tmem_load_20[0], e_off_7, _tmem_load_22[0] * e_dg_7); + float dk2_7 = __fmaf_rn(_tmem_load_21[0], e_off_7, _tmem_load_23[0] * e_dg_7); + dbp_1300[0] = dk2_7 * k_t_7; + float dk2b_7 = dk2_7 * bt_t_7; + float dq_o_7 = dq2_7 + dqf_7; + float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_24[0]; + float qe_6 = q_t_1_7 * e_dg_7; + float kp_18 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_7) * _tmem_load_23[0]); + float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_24[0], k_t_7, cr_6)) + dgf_7; + s_run_1302[0] = s_run_1302[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_15 = t_15 * qrs; - if (t_15 < clen) { + int qoff_7 = t_7 * qrs; + if (t_7 < clen) { { - auto _gst_410 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_410)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_414 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_414)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_414) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_411 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_411)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_415 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_415)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_415) + (0)) = (__nv_bfloat16)dk_o_7; } } float tot_1471 = s_run_1302[0]; @@ -4859,40 +4484,40 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM int b3_1473 = lane_1 >> 3 & 1; int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; - float send_6 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_412 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_412; - float send_7 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_413; - float send_8 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_414 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_414; - float send_9 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_415; + float send_8 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); + float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_416; + float send_9 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_417; + float send_10 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); + float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_418; + float send_11 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, send_11, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_419; float v2_1480[2]; - float send_10 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_416; - float send_11 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_417; + float send_12 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); + float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_420; + float send_13 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, send_13, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_421; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_418; - float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_419; - float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_420; + float _shfl_xor_sync_422 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_422; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_423; + float _shfl_xor_sync_424 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_424; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_421 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_421), "f"(tot_1471) : "memory"); + uint32_t _addr_425 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_425), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4924,132 +4549,132 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_422 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_426 = make_float2(gacc[4], gacc[4]); float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_422), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_425; - asm("abs.f32 %0, %1;" : "=f"(_abs_425) : "f"(rp_z_7.x)); - float _neg_424; - asm("neg.f32 %0, %1;" : "=f"(_neg_424) : "f"(_abs_425)); - float _exp_423 = __nv_expf(_neg_424); - float _abs_428; - asm("abs.f32 %0, %1;" : "=f"(_abs_428) : "f"(rp_z_7.y)); - float _neg_427; - asm("neg.f32 %0, %1;" : "=f"(_neg_427) : "f"(_abs_428)); - float _exp_426 = __nv_expf(_neg_427); - float2 rp_e_7 = make_float2(_exp_423, _exp_426); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_426), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_429; + asm("abs.f32 %0, %1;" : "=f"(_abs_429) : "f"(rp_z_7.x)); + float _neg_428; + asm("neg.f32 %0, %1;" : "=f"(_neg_428) : "f"(_abs_429)); + float _exp_427 = __nv_expf(_neg_428); + float _abs_432; + asm("abs.f32 %0, %1;" : "=f"(_abs_432) : "f"(rp_z_7.y)); + float _neg_431; + asm("neg.f32 %0, %1;" : "=f"(_neg_431) : "f"(_abs_432)); + float _exp_430 = __nv_expf(_neg_431); + float2 rp_e_7 = make_float2(_exp_427, _exp_430); float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_429 = approx_rcp(rp_den_7.x); - float _rcp_430 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_431 = make_float2(_rcp_429, _rcp_430); + float _rcp_433 = approx_rcp(rp_den_7.x); + float _rcp_434 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_435 = make_float2(_rcp_433, _rcp_434); float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_431)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_435)); float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_432 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_436 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_432)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_436)); u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_433 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_437 = make_float2(gacc[4], gacc[4]); float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_433), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_436; - asm("abs.f32 %0, %1;" : "=f"(_abs_436) : "f"(rp_z_8.x)); - float _neg_435; - asm("neg.f32 %0, %1;" : "=f"(_neg_435) : "f"(_abs_436)); - float _exp_434 = __nv_expf(_neg_435); - float _abs_439; - asm("abs.f32 %0, %1;" : "=f"(_abs_439) : "f"(rp_z_8.y)); - float _neg_438; - asm("neg.f32 %0, %1;" : "=f"(_neg_438) : "f"(_abs_439)); - float _exp_437 = __nv_expf(_neg_438); - float2 rp_e_8 = make_float2(_exp_434, _exp_437); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_437), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_440; + asm("abs.f32 %0, %1;" : "=f"(_abs_440) : "f"(rp_z_8.x)); + float _neg_439; + asm("neg.f32 %0, %1;" : "=f"(_neg_439) : "f"(_abs_440)); + float _exp_438 = __nv_expf(_neg_439); + float _abs_443; + asm("abs.f32 %0, %1;" : "=f"(_abs_443) : "f"(rp_z_8.y)); + float _neg_442; + asm("neg.f32 %0, %1;" : "=f"(_neg_442) : "f"(_abs_443)); + float _exp_441 = __nv_expf(_neg_442); + float2 rp_e_8 = make_float2(_exp_438, _exp_441); float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_440 = approx_rcp(rp_den_8.x); - float _rcp_441 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_442 = make_float2(_rcp_440, _rcp_441); + float _rcp_444 = approx_rcp(rp_den_8.x); + float _rcp_445 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_446 = make_float2(_rcp_444, _rcp_445); float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_442)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_446)); float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_443 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_447 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_443)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_447)); u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_444 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_448 = make_float2(gacc[4], gacc[4]); float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_444), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_447; - asm("abs.f32 %0, %1;" : "=f"(_abs_447) : "f"(rp_z_9.x)); - float _neg_446; - asm("neg.f32 %0, %1;" : "=f"(_neg_446) : "f"(_abs_447)); - float _exp_445 = __nv_expf(_neg_446); - float _abs_450; - asm("abs.f32 %0, %1;" : "=f"(_abs_450) : "f"(rp_z_9.y)); - float _neg_449; - asm("neg.f32 %0, %1;" : "=f"(_neg_449) : "f"(_abs_450)); - float _exp_448 = __nv_expf(_neg_449); - float2 rp_e_9 = make_float2(_exp_445, _exp_448); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_448), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_451; + asm("abs.f32 %0, %1;" : "=f"(_abs_451) : "f"(rp_z_9.x)); + float _neg_450; + asm("neg.f32 %0, %1;" : "=f"(_neg_450) : "f"(_abs_451)); + float _exp_449 = __nv_expf(_neg_450); + float _abs_454; + asm("abs.f32 %0, %1;" : "=f"(_abs_454) : "f"(rp_z_9.y)); + float _neg_453; + asm("neg.f32 %0, %1;" : "=f"(_neg_453) : "f"(_abs_454)); + float _exp_452 = __nv_expf(_neg_453); + float2 rp_e_9 = make_float2(_exp_449, _exp_452); float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_451 = approx_rcp(rp_den_9.x); - float _rcp_452 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_453 = make_float2(_rcp_451, _rcp_452); + float _rcp_455 = approx_rcp(rp_den_9.x); + float _rcp_456 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_457 = make_float2(_rcp_455, _rcp_456); float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_453)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_457)); float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_454 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_458 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_454)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_455 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_459 = make_float2(gacc[4], gacc[4]); float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_455), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_458; - asm("abs.f32 %0, %1;" : "=f"(_abs_458) : "f"(rp_z_10.x)); - float _neg_457; - asm("neg.f32 %0, %1;" : "=f"(_neg_457) : "f"(_abs_458)); - float _exp_456 = __nv_expf(_neg_457); - float _abs_461; - asm("abs.f32 %0, %1;" : "=f"(_abs_461) : "f"(rp_z_10.y)); - float _neg_460; - asm("neg.f32 %0, %1;" : "=f"(_neg_460) : "f"(_abs_461)); - float _exp_459 = __nv_expf(_neg_460); - float2 rp_e_10 = make_float2(_exp_456, _exp_459); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_459), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_462; + asm("abs.f32 %0, %1;" : "=f"(_abs_462) : "f"(rp_z_10.x)); + float _neg_461; + asm("neg.f32 %0, %1;" : "=f"(_neg_461) : "f"(_abs_462)); + float _exp_460 = __nv_expf(_neg_461); + float _abs_465; + asm("abs.f32 %0, %1;" : "=f"(_abs_465) : "f"(rp_z_10.y)); + float _neg_464; + asm("neg.f32 %0, %1;" : "=f"(_neg_464) : "f"(_abs_465)); + float _exp_463 = __nv_expf(_neg_464); + float2 rp_e_10 = make_float2(_exp_460, _exp_463); float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_462 = approx_rcp(rp_den_10.x); - float _rcp_463 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_464 = make_float2(_rcp_462, _rcp_463); + float _rcp_466 = approx_rcp(rp_den_10.x); + float _rcp_467 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_468 = make_float2(_rcp_466, _rcp_467); float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_464)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_468)); float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_465 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_469 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_465)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -5064,54 +4689,54 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_466; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_466) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_466; + uint32_t _prmt_470; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_470; int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_467 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_467)); - *(reinterpret_cast(_gst_467) + (0)) = pq_8[0]; + auto _gst_471 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_471)); + *(reinterpret_cast(_gst_471) + (0)) = pq_8[0]; } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_468; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_468) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_468; + uint32_t _prmt_472; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_472; int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_469 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_469)); - *(reinterpret_cast(_gst_469) + (0)) = pq_9[0]; + auto _gst_473 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_473)); + *(reinterpret_cast(_gst_473) + (0)) = pq_9[0]; } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_470; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_470; + uint32_t _prmt_474; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_474) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_474; int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_471 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_471)); - *(reinterpret_cast(_gst_471) + (0)) = pq_10[0]; + auto _gst_475 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_475)); + *(reinterpret_cast(_gst_475) + (0)) = pq_10[0]; } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_472; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_472; + uint32_t _prmt_476; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_476) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_476; int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_473 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_473)); - *(reinterpret_cast(_gst_473) + (0)) = pq_11[0]; + auto _gst_477 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_477)); + *(reinterpret_cast(_gst_477) + (0)) = pq_11[0]; } } int t_a_16 = 16 + 8 * th; @@ -5176,47 +4801,47 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_474; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_474) : "f"(s_a_1), "f"(nm_t_3)); + float _sub_478; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_478) : "f"(s_a_1), "f"(nm_t_3)); float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_474), "f"(pa_1)); - float _sub_475; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_475) : "f"(pa_1), "f"(nm_t_3)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_478), "f"(pa_1)); + float _sub_479; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_479) : "f"(pa_1), "f"(nm_t_3)); float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_475), "f"(s_a_1)); - float _abs_477; - asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(s_a_1)); - float _abs_478; - asm("abs.f32 %0, %1;" : "=f"(_abs_478) : "f"(pa_1)); - uint32_t _setp_476_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_476_bits) : "f"(_abs_477), "f"(_abs_478)); - bool _setp_476 = _setp_476_bits != 0; - float nm_err_3 = ((_setp_476) ? nm_es_3 : nm_ex_3); - float _add_479; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_479) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_479; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_479), "f"(s_a_1)); + float _abs_481; + asm("abs.f32 %0, %1;" : "=f"(_abs_481) : "f"(s_a_1)); + float _abs_482; + asm("abs.f32 %0, %1;" : "=f"(_abs_482) : "f"(pa_1)); + uint32_t _setp_480_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_480_bits) : "f"(_abs_481), "f"(_abs_482)); + bool _setp_480 = _setp_480_bits != 0; + float nm_err_3 = ((_setp_480) ? nm_es_3 : nm_ex_3); + float _add_483; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_483) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_483; s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_480; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_480) : "f"(s_b_1), "f"(nm_t_4)); + float _sub_484; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_484) : "f"(s_b_1), "f"(nm_t_4)); float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_480), "f"(pb_1)); - float _sub_481; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_481) : "f"(pb_1), "f"(nm_t_4)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_484), "f"(pb_1)); + float _sub_485; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_485) : "f"(pb_1), "f"(nm_t_4)); float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_481), "f"(s_b_1)); - float _abs_483; - asm("abs.f32 %0, %1;" : "=f"(_abs_483) : "f"(s_b_1)); - float _abs_484; - asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(pb_1)); - uint32_t _setp_482_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_482_bits) : "f"(_abs_483), "f"(_abs_484)); - bool _setp_482 = _setp_482_bits != 0; - float nm_err_4 = ((_setp_482) ? nm_es_4 : nm_ex_4); - float _add_485; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_485) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_485; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_485), "f"(s_b_1)); + float _abs_487; + asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(s_b_1)); + float _abs_488; + asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(pb_1)); + uint32_t _setp_486_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_486_bits) : "f"(_abs_487), "f"(_abs_488)); + bool _setp_486 = _setp_486_bits != 0; + float nm_err_4 = ((_setp_486) ? nm_es_4 : nm_ex_4); + float _add_489; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_489) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_489; s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5224,8 +4849,8 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_486 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_486), "f"(off_acc[0]) : "memory"); + uint32_t _addr_490 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_490), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5249,9 +4874,9 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1514 = r_322[0]; + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1514 = r_280[0]; float gc_1515 = gn_1514 - 21.640425f; int eb_1516 = 98304 + th * -32768; int ew_1517 = eb_1516 >> 2; @@ -5276,348 +4901,388 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1524[1]; s_run_1524[0] = 0.0f; - int t_16 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1517 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1517 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1517 + 2944 + d_lane]; - float bt_t_16 = btv_1523[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1514); - float e_dg_16 = approx_exp2(g_t_16 - gc_1515); - float dq2_16 = _tmem_load_26[7] * e_dg_16; - float dk2_16 = _tmem_load_27[7] * e_dg_16; - dbp_1522[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_28[7]; - float dg_o_16 = __fmaf_rn(q_t_1_24, dq2_16, (dk2b_16 - _tmem_load_28[7]) * k_t_16) + dgf_16; - s_run_1524[0] = s_run_1524[0] + dg_o_16; + int t_8 = 8 * th + 7; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_8 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_8 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_8 = __uint_as_float(r_283[0] << 16); + float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); + float dqf_8 = pool_f32[ew_1517 + 896 + d_lane]; + float dkf_8 = pool_f32[ew_1517 + 1920 + d_lane]; + float dgf_8 = pool_f32[ew_1517 + 2944 + d_lane]; + float bt_t_8 = btv_1523[7]; + float e_off_8 = approx_exp2(g_t_8 - gn_1514); + float e_dg_8 = approx_exp2(g_t_8 - gc_1515); + float dq2_8 = _tmem_load_26[7] * e_dg_8; + float dk2_8 = _tmem_load_27[7] * e_dg_8; + dbp_1522[7] = dk2_8 * k_t_8; + float dk2b_8 = dk2_8 * bt_t_8; + float dq_o_8 = dq2_8 + dqf_8; + float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_28[7]; + float qe_7 = q_t_1_8 * e_dg_8; + float kp_19 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_8) * _tmem_load_27[7]); + float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_28[7], k_t_8, cr_7)) + dgf_8; + s_run_1524[0] = s_run_1524[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_16 = t_16 * qrs; - if (t_16 < clen) { + int qoff_8 = t_8 * qrs; + if (t_8 < clen) { { - auto _gst_487 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_487)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_487) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_491 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_491)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_488 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_488)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_488) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_492 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_492)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_8; } } - int t_17 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1517 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1517 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1517 + 2816 + d_lane]; - float bt_t_17 = btv_1523[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1514); - float e_dg_17 = approx_exp2(g_t_17 - gc_1515); - float dq2_17 = _tmem_load_26[6] * e_dg_17; - float dk2_17 = _tmem_load_27[6] * e_dg_17; - dbp_1522[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[6]; - float dg_o_17 = __fmaf_rn(q_t_1_25, dq2_17, (dk2b_17 - _tmem_load_28[6]) * k_t_17) + dgf_17; - s_run_1524[0] = s_run_1524[0] + dg_o_17; + int t_9 = 8 * th + 6; + float r_284[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_9 = r_284[0]; + unsigned int r_285[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_9 = __uint_as_float(r_285[0] << 16); + unsigned int r_286[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_9 = __uint_as_float(r_286[0] << 16); + float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); + float dqf_9 = pool_f32[ew_1517 + 768 + d_lane]; + float dkf_9 = pool_f32[ew_1517 + 1792 + d_lane]; + float dgf_9 = pool_f32[ew_1517 + 2816 + d_lane]; + float bt_t_9 = btv_1523[6]; + float e_off_9 = approx_exp2(g_t_9 - gn_1514); + float e_dg_9 = approx_exp2(g_t_9 - gc_1515); + float dq2_9 = _tmem_load_26[6] * e_dg_9; + float dk2_9 = _tmem_load_27[6] * e_dg_9; + dbp_1522[6] = dk2_9 * k_t_9; + float dk2b_9 = dk2_9 * bt_t_9; + float dq_o_9 = dq2_9 + dqf_9; + float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_28[6]; + float qe_8 = q_t_1_9 * e_dg_9; + float kp_20 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_9) * _tmem_load_27[6]); + float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_28[6], k_t_9, cr_8)) + dgf_9; + s_run_1524[0] = s_run_1524[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_17 = t_17 * qrs; - if (t_17 < clen) { + int qoff_9 = t_9 * qrs; + if (t_9 < clen) { { - auto _gst_489 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_489)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_489) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_493 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_493)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_490 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_490)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_490) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_494 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_494)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_9; } } - int t_18 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1517 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1517 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1517 + 2688 + d_lane]; - float bt_t_18 = btv_1523[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1514); - float e_dg_18 = approx_exp2(g_t_18 - gc_1515); - float dq2_18 = _tmem_load_26[5] * e_dg_18; - float dk2_18 = _tmem_load_27[5] * e_dg_18; - dbp_1522[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[5]; - float dg_o_18 = __fmaf_rn(q_t_1_26, dq2_18, (dk2b_18 - _tmem_load_28[5]) * k_t_18) + dgf_18; - s_run_1524[0] = s_run_1524[0] + dg_o_18; + int t_10 = 8 * th + 5; + float r_287[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_10 = r_287[0]; + unsigned int r_288[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_10 = __uint_as_float(r_288[0] << 16); + unsigned int r_289[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_10 = __uint_as_float(r_289[0] << 16); + float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); + float dqf_10 = pool_f32[ew_1517 + 640 + d_lane]; + float dkf_10 = pool_f32[ew_1517 + 1664 + d_lane]; + float dgf_10 = pool_f32[ew_1517 + 2688 + d_lane]; + float bt_t_10 = btv_1523[5]; + float e_off_10 = approx_exp2(g_t_10 - gn_1514); + float e_dg_10 = approx_exp2(g_t_10 - gc_1515); + float dq2_10 = _tmem_load_26[5] * e_dg_10; + float dk2_10 = _tmem_load_27[5] * e_dg_10; + dbp_1522[5] = dk2_10 * k_t_10; + float dk2b_10 = dk2_10 * bt_t_10; + float dq_o_10 = dq2_10 + dqf_10; + float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_28[5]; + float qe_9 = q_t_1_10 * e_dg_10; + float kp_21 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_10) * _tmem_load_27[5]); + float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_28[5], k_t_10, cr_9)) + dgf_10; + s_run_1524[0] = s_run_1524[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_18 = t_18 * qrs; - if (t_18 < clen) { + int qoff_10 = t_10 * qrs; + if (t_10 < clen) { { - auto _gst_491 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_491)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_495 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_495)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_492 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_492)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_496 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_496)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_10; } } - int t_19 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1517 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1517 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1517 + 2560 + d_lane]; - float bt_t_19 = btv_1523[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1514); - float e_dg_19 = approx_exp2(g_t_19 - gc_1515); - float dq2_19 = _tmem_load_26[4] * e_dg_19; - float dk2_19 = _tmem_load_27[4] * e_dg_19; - dbp_1522[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[4]; - float dg_o_19 = __fmaf_rn(q_t_1_27, dq2_19, (dk2b_19 - _tmem_load_28[4]) * k_t_19) + dgf_19; - s_run_1524[0] = s_run_1524[0] + dg_o_19; + int t_11 = 8 * th + 4; + float r_290[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_11 = r_290[0]; + unsigned int r_291[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_11 = __uint_as_float(r_291[0] << 16); + unsigned int r_292[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_11 = __uint_as_float(r_292[0] << 16); + float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); + float dqf_11 = pool_f32[ew_1517 + 512 + d_lane]; + float dkf_11 = pool_f32[ew_1517 + 1536 + d_lane]; + float dgf_11 = pool_f32[ew_1517 + 2560 + d_lane]; + float bt_t_11 = btv_1523[4]; + float e_off_11 = approx_exp2(g_t_11 - gn_1514); + float e_dg_11 = approx_exp2(g_t_11 - gc_1515); + float dq2_11 = _tmem_load_26[4] * e_dg_11; + float dk2_11 = _tmem_load_27[4] * e_dg_11; + dbp_1522[4] = dk2_11 * k_t_11; + float dk2b_11 = dk2_11 * bt_t_11; + float dq_o_11 = dq2_11 + dqf_11; + float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_28[4]; + float qe_10 = q_t_1_11 * e_dg_11; + float kp_22 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_11) * _tmem_load_27[4]); + float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_28[4], k_t_11, cr_10)) + dgf_11; + s_run_1524[0] = s_run_1524[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_19 = t_19 * qrs; - if (t_19 < clen) { + int qoff_11 = t_11 * qrs; + if (t_11 < clen) { { - auto _gst_493 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_493)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_497 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_497)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_494 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_494)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_498 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_498)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_11; } } - int t_20 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1517 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1517 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1517 + 2432 + d_lane]; - float bt_t_20 = btv_1523[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1514); - float e_dg_20 = approx_exp2(g_t_20 - gc_1515); - float dq2_20 = _tmem_load_26[3] * e_dg_20; - float dk2_20 = _tmem_load_27[3] * e_dg_20; - dbp_1522[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[3]; - float dg_o_20 = __fmaf_rn(q_t_1_28, dq2_20, (dk2b_20 - _tmem_load_28[3]) * k_t_20) + dgf_20; - s_run_1524[0] = s_run_1524[0] + dg_o_20; + int t_12 = 8 * th + 3; + float r_293[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_12 = r_293[0]; + unsigned int r_294[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_12 = __uint_as_float(r_294[0] << 16); + unsigned int r_295[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_12 = __uint_as_float(r_295[0] << 16); + float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); + float dqf_12 = pool_f32[ew_1517 + 384 + d_lane]; + float dkf_12 = pool_f32[ew_1517 + 1408 + d_lane]; + float dgf_12 = pool_f32[ew_1517 + 2432 + d_lane]; + float bt_t_12 = btv_1523[3]; + float e_off_12 = approx_exp2(g_t_12 - gn_1514); + float e_dg_12 = approx_exp2(g_t_12 - gc_1515); + float dq2_12 = _tmem_load_26[3] * e_dg_12; + float dk2_12 = _tmem_load_27[3] * e_dg_12; + dbp_1522[3] = dk2_12 * k_t_12; + float dk2b_12 = dk2_12 * bt_t_12; + float dq_o_12 = dq2_12 + dqf_12; + float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_28[3]; + float qe_11 = q_t_1_12 * e_dg_12; + float kp_23 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_12) * _tmem_load_27[3]); + float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_28[3], k_t_12, cr_11)) + dgf_12; + s_run_1524[0] = s_run_1524[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_20 = t_20 * qrs; - if (t_20 < clen) { + int qoff_12 = t_12 * qrs; + if (t_12 < clen) { { - auto _gst_495 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_495)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_499 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_499)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_496 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_496)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_500 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_500)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_12; } } - int t_21 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1517 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1517 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1517 + 2304 + d_lane]; - float bt_t_21 = btv_1523[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1514); - float e_dg_21 = approx_exp2(g_t_21 - gc_1515); - float dq2_21 = _tmem_load_26[2] * e_dg_21; - float dk2_21 = _tmem_load_27[2] * e_dg_21; - dbp_1522[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[2]; - float dg_o_21 = __fmaf_rn(q_t_1_29, dq2_21, (dk2b_21 - _tmem_load_28[2]) * k_t_21) + dgf_21; - s_run_1524[0] = s_run_1524[0] + dg_o_21; + int t_13 = 8 * th + 2; + float r_296[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_13 = r_296[0]; + unsigned int r_297[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_13 = __uint_as_float(r_297[0] << 16); + unsigned int r_298[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_13 = __uint_as_float(r_298[0] << 16); + float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); + float dqf_13 = pool_f32[ew_1517 + 256 + d_lane]; + float dkf_13 = pool_f32[ew_1517 + 1280 + d_lane]; + float dgf_13 = pool_f32[ew_1517 + 2304 + d_lane]; + float bt_t_13 = btv_1523[2]; + float e_off_13 = approx_exp2(g_t_13 - gn_1514); + float e_dg_13 = approx_exp2(g_t_13 - gc_1515); + float dq2_13 = _tmem_load_26[2] * e_dg_13; + float dk2_13 = _tmem_load_27[2] * e_dg_13; + dbp_1522[2] = dk2_13 * k_t_13; + float dk2b_13 = dk2_13 * bt_t_13; + float dq_o_13 = dq2_13 + dqf_13; + float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_28[2]; + float qe_12 = q_t_1_13 * e_dg_13; + float kp_24 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_13) * _tmem_load_27[2]); + float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_28[2], k_t_13, cr_12)) + dgf_13; + s_run_1524[0] = s_run_1524[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_21 = t_21 * qrs; - if (t_21 < clen) { + int qoff_13 = t_13 * qrs; + if (t_13 < clen) { { - auto _gst_497 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_497)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_501 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_501)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_498 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_498)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_502 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_502)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_13; } } - int t_22 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1517 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1517 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1517 + 2176 + d_lane]; - float bt_t_22 = btv_1523[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1514); - float e_dg_22 = approx_exp2(g_t_22 - gc_1515); - float dq2_22 = _tmem_load_26[1] * e_dg_22; - float dk2_22 = _tmem_load_27[1] * e_dg_22; - dbp_1522[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[1]; - float dg_o_22 = __fmaf_rn(q_t_1_30, dq2_22, (dk2b_22 - _tmem_load_28[1]) * k_t_22) + dgf_22; - s_run_1524[0] = s_run_1524[0] + dg_o_22; + int t_14 = 8 * th + 1; + float r_299[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_14 = r_299[0]; + unsigned int r_300[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_14 = __uint_as_float(r_300[0] << 16); + unsigned int r_301[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_14 = __uint_as_float(r_301[0] << 16); + float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); + float dqf_14 = pool_f32[ew_1517 + 128 + d_lane]; + float dkf_14 = pool_f32[ew_1517 + 1152 + d_lane]; + float dgf_14 = pool_f32[ew_1517 + 2176 + d_lane]; + float bt_t_14 = btv_1523[1]; + float e_off_14 = approx_exp2(g_t_14 - gn_1514); + float e_dg_14 = approx_exp2(g_t_14 - gc_1515); + float dq2_14 = _tmem_load_26[1] * e_dg_14; + float dk2_14 = _tmem_load_27[1] * e_dg_14; + dbp_1522[1] = dk2_14 * k_t_14; + float dk2b_14 = dk2_14 * bt_t_14; + float dq_o_14 = dq2_14 + dqf_14; + float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_28[1]; + float qe_13 = q_t_1_14 * e_dg_14; + float kp_25 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_14) * _tmem_load_27[1]); + float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_28[1], k_t_14, cr_13)) + dgf_14; + s_run_1524[0] = s_run_1524[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_22 = t_22 * qrs; - if (t_22 < clen) { + int qoff_14 = t_14 * qrs; + if (t_14 < clen) { { - auto _gst_499 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_499)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_503 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_503)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_503) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_500 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_500)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_504 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_504)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_504) + (0)) = (__nv_bfloat16)dk_o_14; } } - int t_23 = 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1517 + d_lane]; - float dkf_23 = pool_f32[ew_1517 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1517 + 2048 + d_lane]; - float bt_t_23 = btv_1523[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1514); - float e_dg_23 = approx_exp2(g_t_23 - gc_1515); - float dq2_23 = _tmem_load_26[0] * e_dg_23; - float dk2_23 = _tmem_load_27[0] * e_dg_23; - dbp_1522[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[0]; - float dg_o_23 = __fmaf_rn(q_t_1_31, dq2_23, (dk2b_23 - _tmem_load_28[0]) * k_t_23) + dgf_23; - s_run_1524[0] = s_run_1524[0] + dg_o_23; + int t_15 = 8 * th; + float r_302[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_15 = r_302[0]; + unsigned int r_303[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_15 = __uint_as_float(r_303[0] << 16); + unsigned int r_304[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_15 = __uint_as_float(r_304[0] << 16); + float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); + float dqf_15 = pool_f32[ew_1517 + d_lane]; + float dkf_15 = pool_f32[ew_1517 + 1024 + d_lane]; + float dgf_15 = pool_f32[ew_1517 + 2048 + d_lane]; + float bt_t_15 = btv_1523[0]; + float e_off_15 = approx_exp2(g_t_15 - gn_1514); + float e_dg_15 = approx_exp2(g_t_15 - gc_1515); + float dq2_15 = _tmem_load_26[0] * e_dg_15; + float dk2_15 = _tmem_load_27[0] * e_dg_15; + dbp_1522[0] = dk2_15 * k_t_15; + float dk2b_15 = dk2_15 * bt_t_15; + float dq_o_15 = dq2_15 + dqf_15; + float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_28[0]; + float qe_14 = q_t_1_15 * e_dg_15; + float kp_26 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_15) * _tmem_load_27[0]); + float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_28[0], k_t_15, cr_14)) + dgf_15; + s_run_1524[0] = s_run_1524[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_23 = t_23 * qrs; - if (t_23 < clen) { + int qoff_15 = t_15 * qrs; + if (t_15 < clen) { { - auto _gst_501 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_501)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_505 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_505)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_505) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_502 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_502)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_506 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_506)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_506) + (0)) = (__nv_bfloat16)dk_o_15; } } float tot_1693 = s_run_1524[0]; @@ -5625,38 +5290,38 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM int b3_1695 = lane_1 >> 3 & 1; int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; - float send_12 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_503 = __shfl_xor_sync(0xFFFFFFFF, send_12, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_503; - float send_13 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_504; - float send_14 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_505 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_505; - float send_15 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_506; + float send_14 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); + float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_507; + float send_15 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); + float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_508; + float send_16 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); + float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_509; + float send_17 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); + float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, send_17, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_510; float v2_1702[2]; - float send_16 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_16, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_507; - float send_17 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_508; + float send_18 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); + float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_511; + float send_19 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); + float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, send_19, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_512; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_509; - float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_510; - float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_511; + float _shfl_xor_sync_513 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_513; + float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_514; + float _shfl_xor_sync_515 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_515; tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_512 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_512), "f"(tot_1693) : "memory"); + uint32_t _addr_516 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_516), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5688,132 +5353,132 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_513 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_517 = make_float2(gacc[4], gacc[4]); float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_513), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_516; - asm("abs.f32 %0, %1;" : "=f"(_abs_516) : "f"(rp_z_11.x)); - float _neg_515; - asm("neg.f32 %0, %1;" : "=f"(_neg_515) : "f"(_abs_516)); - float _exp_514 = __nv_expf(_neg_515); - float _abs_519; - asm("abs.f32 %0, %1;" : "=f"(_abs_519) : "f"(rp_z_11.y)); - float _neg_518; - asm("neg.f32 %0, %1;" : "=f"(_neg_518) : "f"(_abs_519)); - float _exp_517 = __nv_expf(_neg_518); - float2 rp_e_11 = make_float2(_exp_514, _exp_517); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_517), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_520; + asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_11.x)); + float _neg_519; + asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); + float _exp_518 = __nv_expf(_neg_519); + float _abs_523; + asm("abs.f32 %0, %1;" : "=f"(_abs_523) : "f"(rp_z_11.y)); + float _neg_522; + asm("neg.f32 %0, %1;" : "=f"(_neg_522) : "f"(_abs_523)); + float _exp_521 = __nv_expf(_neg_522); + float2 rp_e_11 = make_float2(_exp_518, _exp_521); float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_520 = approx_rcp(rp_den_11.x); - float _rcp_521 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_522 = make_float2(_rcp_520, _rcp_521); + float _rcp_524 = approx_rcp(rp_den_11.x); + float _rcp_525 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_526 = make_float2(_rcp_524, _rcp_525); float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_522)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_526)); float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_523 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_527 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_527)); u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_524 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_528 = make_float2(gacc[4], gacc[4]); float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_524), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_527; - asm("abs.f32 %0, %1;" : "=f"(_abs_527) : "f"(rp_z_12.x)); - float _neg_526; - asm("neg.f32 %0, %1;" : "=f"(_neg_526) : "f"(_abs_527)); - float _exp_525 = __nv_expf(_neg_526); - float _abs_530; - asm("abs.f32 %0, %1;" : "=f"(_abs_530) : "f"(rp_z_12.y)); - float _neg_529; - asm("neg.f32 %0, %1;" : "=f"(_neg_529) : "f"(_abs_530)); - float _exp_528 = __nv_expf(_neg_529); - float2 rp_e_12 = make_float2(_exp_525, _exp_528); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_528), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_531; + asm("abs.f32 %0, %1;" : "=f"(_abs_531) : "f"(rp_z_12.x)); + float _neg_530; + asm("neg.f32 %0, %1;" : "=f"(_neg_530) : "f"(_abs_531)); + float _exp_529 = __nv_expf(_neg_530); + float _abs_534; + asm("abs.f32 %0, %1;" : "=f"(_abs_534) : "f"(rp_z_12.y)); + float _neg_533; + asm("neg.f32 %0, %1;" : "=f"(_neg_533) : "f"(_abs_534)); + float _exp_532 = __nv_expf(_neg_533); + float2 rp_e_12 = make_float2(_exp_529, _exp_532); float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_531 = approx_rcp(rp_den_12.x); - float _rcp_532 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_533 = make_float2(_rcp_531, _rcp_532); + float _rcp_535 = approx_rcp(rp_den_12.x); + float _rcp_536 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_537 = make_float2(_rcp_535, _rcp_536); float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_533)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_537)); float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_534 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_538 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_534)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_538)); u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_535 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_539 = make_float2(gacc[4], gacc[4]); float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_535), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_538; - asm("abs.f32 %0, %1;" : "=f"(_abs_538) : "f"(rp_z_13.x)); - float _neg_537; - asm("neg.f32 %0, %1;" : "=f"(_neg_537) : "f"(_abs_538)); - float _exp_536 = __nv_expf(_neg_537); - float _abs_541; - asm("abs.f32 %0, %1;" : "=f"(_abs_541) : "f"(rp_z_13.y)); - float _neg_540; - asm("neg.f32 %0, %1;" : "=f"(_neg_540) : "f"(_abs_541)); - float _exp_539 = __nv_expf(_neg_540); - float2 rp_e_13 = make_float2(_exp_536, _exp_539); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_539), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_542; + asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(rp_z_13.x)); + float _neg_541; + asm("neg.f32 %0, %1;" : "=f"(_neg_541) : "f"(_abs_542)); + float _exp_540 = __nv_expf(_neg_541); + float _abs_545; + asm("abs.f32 %0, %1;" : "=f"(_abs_545) : "f"(rp_z_13.y)); + float _neg_544; + asm("neg.f32 %0, %1;" : "=f"(_neg_544) : "f"(_abs_545)); + float _exp_543 = __nv_expf(_neg_544); + float2 rp_e_13 = make_float2(_exp_540, _exp_543); float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_542 = approx_rcp(rp_den_13.x); - float _rcp_543 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_544 = make_float2(_rcp_542, _rcp_543); + float _rcp_546 = approx_rcp(rp_den_13.x); + float _rcp_547 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_548 = make_float2(_rcp_546, _rcp_547); float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_544)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_548)); float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_545 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_549 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_545)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_549)); u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_546 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_550 = make_float2(gacc[4], gacc[4]); float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_546), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_549; - asm("abs.f32 %0, %1;" : "=f"(_abs_549) : "f"(rp_z_14.x)); - float _neg_548; - asm("neg.f32 %0, %1;" : "=f"(_neg_548) : "f"(_abs_549)); - float _exp_547 = __nv_expf(_neg_548); - float _abs_552; - asm("abs.f32 %0, %1;" : "=f"(_abs_552) : "f"(rp_z_14.y)); - float _neg_551; - asm("neg.f32 %0, %1;" : "=f"(_neg_551) : "f"(_abs_552)); - float _exp_550 = __nv_expf(_neg_551); - float2 rp_e_14 = make_float2(_exp_547, _exp_550); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_550), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_553; + asm("abs.f32 %0, %1;" : "=f"(_abs_553) : "f"(rp_z_14.x)); + float _neg_552; + asm("neg.f32 %0, %1;" : "=f"(_neg_552) : "f"(_abs_553)); + float _exp_551 = __nv_expf(_neg_552); + float _abs_556; + asm("abs.f32 %0, %1;" : "=f"(_abs_556) : "f"(rp_z_14.y)); + float _neg_555; + asm("neg.f32 %0, %1;" : "=f"(_neg_555) : "f"(_abs_556)); + float _exp_554 = __nv_expf(_neg_555); + float2 rp_e_14 = make_float2(_exp_551, _exp_554); float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_553 = approx_rcp(rp_den_14.x); - float _rcp_554 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_555 = make_float2(_rcp_553, _rcp_554); + float _rcp_557 = approx_rcp(rp_den_14.x); + float _rcp_558 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_559 = make_float2(_rcp_557, _rcp_558); float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_555)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_559)); float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_556 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_560 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_556)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_560)); u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -5828,54 +5493,54 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_557; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_557) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_557; + uint32_t _prmt_561; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_561; int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_558 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_558)); - *(reinterpret_cast(_gst_558) + (0)) = pq_12[0]; + auto _gst_562 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_562)); + *(reinterpret_cast(_gst_562) + (0)) = pq_12[0]; } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_559; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_559) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_559; + uint32_t _prmt_563; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_563; int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_560 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_560)); - *(reinterpret_cast(_gst_560) + (0)) = pq_13[0]; + auto _gst_564 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_564)); + *(reinterpret_cast(_gst_564) + (0)) = pq_13[0]; } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_561; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_561; + uint32_t _prmt_565; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_565) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_565; int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_562 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_562)); - *(reinterpret_cast(_gst_562) + (0)) = pq_14[0]; + auto _gst_566 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_566)); + *(reinterpret_cast(_gst_566) + (0)) = pq_14[0]; } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_563; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_563; + uint32_t _prmt_567; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_567) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_567; int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_564 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_564)); - *(reinterpret_cast(_gst_564) + (0)) = pq_15[0]; + auto _gst_568 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_568)); + *(reinterpret_cast(_gst_568) + (0)) = pq_15[0]; } } int t_a_24 = 8 * th; @@ -5940,47 +5605,47 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_565; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_565) : "f"(s_a_2), "f"(nm_t_5)); + float _sub_569; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_569) : "f"(s_a_2), "f"(nm_t_5)); float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_565), "f"(pa_2)); - float _sub_566; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_566) : "f"(pa_2), "f"(nm_t_5)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_569), "f"(pa_2)); + float _sub_570; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_570) : "f"(pa_2), "f"(nm_t_5)); float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_566), "f"(s_a_2)); - float _abs_568; - asm("abs.f32 %0, %1;" : "=f"(_abs_568) : "f"(s_a_2)); - float _abs_569; - asm("abs.f32 %0, %1;" : "=f"(_abs_569) : "f"(pa_2)); - uint32_t _setp_567_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_567_bits) : "f"(_abs_568), "f"(_abs_569)); - bool _setp_567 = _setp_567_bits != 0; - float nm_err_5 = ((_setp_567) ? nm_es_5 : nm_ex_5); - float _add_570; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_570; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_570), "f"(s_a_2)); + float _abs_572; + asm("abs.f32 %0, %1;" : "=f"(_abs_572) : "f"(s_a_2)); + float _abs_573; + asm("abs.f32 %0, %1;" : "=f"(_abs_573) : "f"(pa_2)); + uint32_t _setp_571_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_571_bits) : "f"(_abs_572), "f"(_abs_573)); + bool _setp_571 = _setp_571_bits != 0; + float nm_err_5 = ((_setp_571) ? nm_es_5 : nm_ex_5); + float _add_574; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_574; s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_571; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(s_b_2), "f"(nm_t_6)); + float _sub_575; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_575) : "f"(s_b_2), "f"(nm_t_6)); float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_571), "f"(pb_2)); - float _sub_572; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_572) : "f"(pb_2), "f"(nm_t_6)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_575), "f"(pb_2)); + float _sub_576; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_576) : "f"(pb_2), "f"(nm_t_6)); float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_572), "f"(s_b_2)); - float _abs_574; - asm("abs.f32 %0, %1;" : "=f"(_abs_574) : "f"(s_b_2)); - float _abs_575; - asm("abs.f32 %0, %1;" : "=f"(_abs_575) : "f"(pb_2)); - uint32_t _setp_573_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_573_bits) : "f"(_abs_574), "f"(_abs_575)); - bool _setp_573 = _setp_573_bits != 0; - float nm_err_6 = ((_setp_573) ? nm_es_6 : nm_ex_6); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_576; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_576), "f"(s_b_2)); + float _abs_578; + asm("abs.f32 %0, %1;" : "=f"(_abs_578) : "f"(s_b_2)); + float _abs_579; + asm("abs.f32 %0, %1;" : "=f"(_abs_579) : "f"(pb_2)); + uint32_t _setp_577_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_577_bits) : "f"(_abs_578), "f"(_abs_579)); + bool _setp_577 = _setp_577_bits != 0; + float nm_err_6 = ((_setp_577) ? nm_es_6 : nm_ex_6); + float _add_580; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_580; s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -6019,60 +5684,60 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_578; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_578) : "f"(ts_s), "f"(ts_bb)); - float _sub_577; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_577) : "f"(s_a_3), "f"(_sub_578)); - float _sub_579; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_579) : "f"(other[0]), "f"(ts_bb)); + float _sub_582; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s), "f"(ts_bb)); + float _sub_581; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(s_a_3), "f"(_sub_582)); + float _sub_583; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(other[0]), "f"(ts_bb)); float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_577), "f"(_sub_579)); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_a_3), "f"(other[1])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_581), "f"(_sub_583)); + float _add_584; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(c_a_3), "f"(other[1])); float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_580), "f"(ts_e)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_584), "f"(ts_e)); float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_582; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_581; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(ts_s), "f"(_sub_582)); - float _sub_583; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(pm_c), "f"(ts_bb_0)); + float _sub_586; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_585; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s), "f"(_sub_586)); + float _sub_587; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_587) : "f"(pm_c), "f"(ts_bb_0)); float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_581), "f"(_sub_583)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_585), "f"(_sub_587)); s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_585; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_584; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_584) : "f"(s_b_3), "f"(_sub_585)); - float _sub_586; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(other[2]), "f"(ts_bb_1)); + float _sub_589; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_588; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(s_b_3), "f"(_sub_589)); + float _sub_590; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(other[2]), "f"(ts_bb_1)); float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_584), "f"(_sub_586)); - float _add_587; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_587) : "f"(c_b_3), "f"(other[3])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_588), "f"(_sub_590)); + float _add_591; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_591) : "f"(c_b_3), "f"(other[3])); float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_587), "f"(ts_e_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_591), "f"(ts_e_1)); float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_589; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_588; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(ts_s_1), "f"(_sub_589)); - float _sub_590; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(pm_c_0), "f"(ts_bb_2)); + float _sub_593; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_593) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_592; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_592) : "f"(ts_s_1), "f"(_sub_593)); + float _sub_594; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_594) : "f"(pm_c_0), "f"(ts_bb_2)); float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_588), "f"(_sub_590)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_592), "f"(_sub_594)); s_b_3 = ts_s_2; c_b_3 = ts_e_2; gacc[0] = s_a_3; @@ -6093,7 +5758,7 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_1706; pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1484; pool_f32[28160 + qd * 64 + 32 + 8 * th + e_star] = v1; - pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_1241; + pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_0; } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync 1, 256;" ::: "memory"); @@ -6183,41 +5848,41 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + for (int r_305 = 0; r_305 < 16; r_305++) { + int t_16 = r_305; + if (t_16 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6289,41 +5954,41 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_24 = 16 + r_348; - if (t_24 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + for (int r_306 = 0; r_306 < 16; r_306++) { + int t_17 = 16 + r_306; + if (t_17 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6439,41 +6104,41 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_25 = 32 + r_349; - if (t_25 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + for (int r_307 = 0; r_307 < 16; r_307++) { + int t_18 = 32 + r_307; + if (t_18 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6621,41 +6286,41 @@ kernel_cake_kda_det_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorM z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_26 = 48 + r_350; - if (t_26 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + for (int r_308 = 0; r_308 < 16; r_308++) { + int t_19 = 48 + r_308; + if (t_19 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); From e246463814749ee34e2644341dc77c0fcb00255b Mon Sep 17 00:00:00 2001 From: Yingyi Huang Date: Thu, 5 Nov 2026 10:27:55 -0800 Subject: [PATCH 2/5] cake_kda_train: re-copy the deterministic glm_intra programs after the diagonal TF32 rounding fix (export kda_train migrate) The parent commit changes four deterministic GLM intra programs in place (same program names, new sources) that the non-deterministic package ships as copies. The copies and their build.json entries follow: det record program copy entry glm_intra 54786ffd01bd9b6bb416 cake_kda_train_..._sm_100a glm_intra_v14 c27b09eb5f5881be2969 cake_kda_train_..._sm_103a glm_intra_v18 aa6b5c23c97e1cb9880a cake_kda_train_..._sm_107a glm_intra_raw ea14e16d1fc71364ba03 cake_kda_train_..._sm_107a Each copy is the deterministic unit with the package prefix renamed; the build.json entries carry the new source digests and identity closures. The other programs of the parent commit (glm_intra_l2n, glm_intra_v30_l2n, glm_intra_l2n_capture, glm_intra_raw_l2n, glm_intra_raw_l2n_capture) belong to the in-kernel Q/K L2 norm class, which this package does not copy. The package's own programs, its registry and the KCP package are unchanged. A second migration run changes nothing (fixed point); the same run on 79cf0557 changes nothing. tests/experimental/test_cake_kda*.py on CPU: 2000 passed, 913 skipped (parent: 1 failed, test_copied_programs_are_the_deterministic_packages_sources). --- .../cake/cake_kda_train/build.json | 16 +- ...e_kda_train_54786ffd01bd9b6bb416_kernel.cu | 1301 +++--- ...e_kda_train_aa6b5c23c97e1cb9880a_kernel.cu | 1293 ++++-- ...e_kda_train_c27b09eb5f5881be2969_kernel.cu | 1321 +++--- ...e_kda_train_ea14e16d1fc71364ba03_kernel.cu | 4071 ++++++++--------- 5 files changed, 4252 insertions(+), 3750 deletions(-) diff --git a/flashinfer/experimental/cake/cake_kda_train/build.json b/flashinfer/experimental/cake/cake_kda_train/build.json index 55bb0c17ac6..912af147f76 100644 --- a/flashinfer/experimental/cake/cake_kda_train/build.json +++ b/flashinfer/experimental/cake/cake_kda_train/build.json @@ -1784,7 +1784,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "ee3919821c64cbbeb3cbd4c8ea6867cfb56af94836af5902624ab51cdd2a35ab", + "closure_sha256": "3429c354d069f942de6181c72a18b28394418e455052ffbf268a849329b32287", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1796,7 +1796,7 @@ "path": "aot/cake_kda_train_54786ffd01bd9b6bb416_sm_100a.so", "sha256": { "csrc/cake_kda_train_54786ffd01bd9b6bb416_binding.cu": "a77d517dc4c55ad6ddcf1797a307fb02d0b67ba8ecdf3d31ca2fd7e7a2de000d", - "csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu": "584354d5bce463661438e5324be10347b08d50037ce3b42feb7d290f30f2a96a" + "csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu": "c5ae178a848262845224ab630925a8aa7fe4d73a223c02eea4100cc5cb179ef5" }, "sources": [ "csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu", @@ -3275,7 +3275,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "c3cbf2d5315b8dd39f4ed945a012e5b7065862b5340f8a9336a9910005d37ff8", + "closure_sha256": "177c731ea041e17af99bfcd7e4de395cfb60fe43df8281974c7a991912975fff", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -3287,7 +3287,7 @@ "path": "aot/cake_kda_train_aa6b5c23c97e1cb9880a_sm_107a.so", "sha256": { "csrc/cake_kda_train_aa6b5c23c97e1cb9880a_binding.cu": "92b299e2442b425315c2a48d87fc8715b414a723516f8155842a95263970385a", - "csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu": "c0b2637c19e401d19bb4c568bb42cdf0d1063272d50cf42f1b8a2a8c685627ac" + "csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu": "c85805e47b1ff2edba5f87fad10e5fc7c8f47ce243a9fc0b665e190d6990b51e" }, "sources": [ "csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu", @@ -3611,7 +3611,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "85d2c2aed90d5bca7837f738fc6aa255fe7dfb7d46aa052925b13a378d3483f0", + "closure_sha256": "736611400f691d1e6b9d21b1a4de5b87d185faf644e7d799db6a0ed89c078964", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -3623,7 +3623,7 @@ "path": "aot/cake_kda_train_c27b09eb5f5881be2969_sm_103a.so", "sha256": { "csrc/cake_kda_train_c27b09eb5f5881be2969_binding.cu": "72b08e731d61855c9b753d98f4f3303dc2f4cd5efbac1ed1be0443f75da02b05", - "csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu": "37a294a92553fd7497de5fafbcf6b593aeb0af772dcb066570fdb10eb3724553" + "csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu": "eb973be56b0c1e6834d5bb0a0a76394cb04e50b5810e6474f4436202cc79fbb4" }, "sources": [ "csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu", @@ -4325,7 +4325,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "20d25a83809d70c90373aa7bcec3cee701047767d6748753f9e47d6ab99b8c32", + "closure_sha256": "db14d39a4cbab2729c1464fc1aa1d14459ffc70e06d169d92e8cb2e4337c46af", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4337,7 +4337,7 @@ "path": "aot/cake_kda_train_ea14e16d1fc71364ba03_sm_107a.so", "sha256": { "csrc/cake_kda_train_ea14e16d1fc71364ba03_binding.cu": "5f19248aea28a42fb7c6e76ab607844338ea1e51c9f61b392dc47ed6a3d63c54", - "csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu": "1f507cffec0c6be6a1d33573c5e0d56ac14ea13fcbc63195e72373e9652355fe" + "csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu": "d97b83f275a6f9d6fd8bd229923f8c4e5c19e55a55c075cd7e7d1489760f5140" }, "sources": [ "csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu", diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu index 4a94bfa903d..ac525b966de 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_54786ffd01bd9b6bb416_kernel.cu @@ -130,6 +130,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -172,6 +179,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2235,15 +2258,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2264,15 +2293,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2293,15 +2328,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2322,15 +2363,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2366,12 +2413,18 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2391,12 +2444,18 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2416,12 +2475,18 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2441,12 +2506,18 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2491,15 +2562,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2520,15 +2597,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2549,15 +2632,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2578,15 +2667,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2625,15 +2720,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2654,15 +2755,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2683,15 +2790,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2712,15 +2825,21 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2791,37 +2910,37 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2833,29 +2952,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2863,29 +2982,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2953,24 +3072,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; - float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0) + dgf_0; + float qe = q_t0_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr)) + dgf_0; int goff_0 = t_0 * rs_g; int qoff_0 = t_0 * qrs; { - auto _gst_216 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast(_gst_216) + (0)) = dg_o_0; + auto _gst_248 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast(_gst_248) + (0)) = dg_o_0; } if (t_0 < clen) { { - auto _gst_217 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_249 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_218 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_250 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -2996,24 +3120,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_1 = dk2_1 * bt_t_1; float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; - float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1) + dgf_1; + float qe_0 = q_t0_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0)) + dgf_1; int goff_1 = t_1 * rs_g; int qoff_1 = t_1 * qrs; { - auto _gst_219 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast(_gst_219) + (0)) = dg_o_1; + auto _gst_251 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast(_gst_251) + (0)) = dg_o_1; } if (t_1 < clen) { { - auto _gst_220 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_252 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_221 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_253 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3039,24 +3168,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; - float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2) + dgf_2; + float qe_1 = q_t0_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1)) + dgf_2; int goff_2 = t_2 * rs_g; int qoff_2 = t_2 * qrs; { - auto _gst_222 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast(_gst_222) + (0)) = dg_o_2; + auto _gst_254 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast(_gst_254) + (0)) = dg_o_2; } if (t_2 < clen) { { - auto _gst_223 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_255 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_224 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_256 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3082,24 +3216,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; - float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3) + dgf_3; + float qe_2 = q_t0_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2)) + dgf_3; int goff_3 = t_3 * rs_g; int qoff_3 = t_3 * qrs; { - auto _gst_225 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast(_gst_225) + (0)) = dg_o_3; + auto _gst_257 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast(_gst_257) + (0)) = dg_o_3; } if (t_3 < clen) { { - auto _gst_226 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_258 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_227 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_259 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3125,24 +3264,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; - float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4) + dgf_4; + float qe_3 = q_t0_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3)) + dgf_4; int goff_4 = t_4 * rs_g; int qoff_4 = t_4 * qrs; { - auto _gst_228 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast(_gst_228) + (0)) = dg_o_4; + auto _gst_260 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast(_gst_260) + (0)) = dg_o_4; } if (t_4 < clen) { { - auto _gst_229 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_261 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_230 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_262 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3168,24 +3312,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; - float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5) + dgf_5; + float qe_4 = q_t0_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4)) + dgf_5; int goff_5 = t_5 * rs_g; int qoff_5 = t_5 * qrs; { - auto _gst_231 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast(_gst_231) + (0)) = dg_o_5; + auto _gst_263 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast(_gst_263) + (0)) = dg_o_5; } if (t_5 < clen) { { - auto _gst_232 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_232)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_232) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_264 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_264)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_264) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_233 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_233)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_233) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_265 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_265)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3211,24 +3360,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; - float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6) + dgf_6; + float qe_5 = q_t0_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5)) + dgf_6; int goff_6 = t_6 * rs_g; int qoff_6 = t_6 * qrs; { - auto _gst_234 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_234)); - *(reinterpret_cast(_gst_234) + (0)) = dg_o_6; + auto _gst_266 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_266)); + *(reinterpret_cast(_gst_266) + (0)) = dg_o_6; } if (t_6 < clen) { { - auto _gst_235 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_235)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_235) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_267 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_267)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_267) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_236 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_236)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_236) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_268 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_268)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3254,24 +3408,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; - float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7) + dgf_7; + float qe_6 = q_t0_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6)) + dgf_7; int goff_7 = t_7 * rs_g; int qoff_7 = t_7 * qrs; { - auto _gst_237 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_237)); - *(reinterpret_cast(_gst_237) + (0)) = dg_o_7; + auto _gst_269 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_269)); + *(reinterpret_cast(_gst_269) + (0)) = dg_o_7; } if (t_7 < clen) { { - auto _gst_238 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_238)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_238) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_270 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_270)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_270) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_239 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_239)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_239) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_271 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_271)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dk_o_7; } } int b4 = lane_1 >> 4 & 1; @@ -3279,31 +3438,31 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_240; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_272; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_241 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_241; + float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_273; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_242 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_242; + float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_274; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_243 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_243; + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_275; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_244 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_244; + float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_276; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_245 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_245; + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_277; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_246 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_246; - float _shfl_xor_sync_247 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_247; - float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_248; + float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_278; + float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_279; + float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_280; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3361,24 +3520,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; - float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8) + dgf_8; + float qe_7 = q_t0_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7)) + dgf_8; int goff_8 = t_8 * rs_g; int qoff_8 = t_8 * qrs; { - auto _gst_249 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast(_gst_249) + (0)) = dg_o_8; + auto _gst_281 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast(_gst_281) + (0)) = dg_o_8; } if (t_8 < clen) { { - auto _gst_250 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_282 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_251 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_283 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3404,24 +3568,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; - float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9) + dgf_9; + float qe_8 = q_t0_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8)) + dgf_9; int goff_9 = t_9 * rs_g; int qoff_9 = t_9 * qrs; { - auto _gst_252 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast(_gst_252) + (0)) = dg_o_9; + auto _gst_284 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast(_gst_284) + (0)) = dg_o_9; } if (t_9 < clen) { { - auto _gst_253 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_285 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_254 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_286 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3447,24 +3616,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; - float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10) + dgf_10; + float qe_9 = q_t0_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9)) + dgf_10; int goff_10 = t_10 * rs_g; int qoff_10 = t_10 * qrs; { - auto _gst_255 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast(_gst_255) + (0)) = dg_o_10; + auto _gst_287 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast(_gst_287) + (0)) = dg_o_10; } if (t_10 < clen) { { - auto _gst_256 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_288 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_257 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_257)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_289 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_289)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3490,24 +3664,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; - float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11) + dgf_11; + float qe_10 = q_t0_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10)) + dgf_11; int goff_11 = t_11 * rs_g; int qoff_11 = t_11 * qrs; { - auto _gst_258 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_258)); - *(reinterpret_cast(_gst_258) + (0)) = dg_o_11; + auto _gst_290 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_290)); + *(reinterpret_cast(_gst_290) + (0)) = dg_o_11; } if (t_11 < clen) { { - auto _gst_259 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_259)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_291 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_291)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_260 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_260)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_292 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_292)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3533,24 +3712,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; - float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12) + dgf_12; + float qe_11 = q_t0_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11)) + dgf_12; int goff_12 = t_12 * rs_g; int qoff_12 = t_12 * qrs; { - auto _gst_261 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_261)); - *(reinterpret_cast(_gst_261) + (0)) = dg_o_12; + auto _gst_293 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_293)); + *(reinterpret_cast(_gst_293) + (0)) = dg_o_12; } if (t_12 < clen) { { - auto _gst_262 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_262)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_294 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_294)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_263 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_263)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_295 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_295)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3576,24 +3760,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; - float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13) + dgf_13; + float qe_12 = q_t0_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12)) + dgf_13; int goff_13 = t_13 * rs_g; int qoff_13 = t_13 * qrs; { - auto _gst_264 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_264)); - *(reinterpret_cast(_gst_264) + (0)) = dg_o_13; + auto _gst_296 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_296)); + *(reinterpret_cast(_gst_296) + (0)) = dg_o_13; } if (t_13 < clen) { { - auto _gst_265 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_265)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_297 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_297)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_266 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_266)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_266) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_298 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_298)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3619,24 +3808,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; - float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14) + dgf_14; + float qe_13 = q_t0_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13)) + dgf_14; int goff_14 = t_14 * rs_g; int qoff_14 = t_14 * qrs; { - auto _gst_267 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_o_14; + auto _gst_299 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_o_14; } if (t_14 < clen) { { - auto _gst_268 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_300 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_300) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_269 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_269) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_301 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3662,24 +3856,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; - float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15) + dgf_15; + float qe_14 = q_t0_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14)) + dgf_15; int goff_15 = t_15 * rs_g; int qoff_15 = t_15 * qrs; { - auto _gst_270 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_o_15; + auto _gst_302 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_o_15; } if (t_15 < clen) { { - auto _gst_271 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_303 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_303) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_272 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_272) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_304 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dk_o_15; } } int b4_0 = lane_1 >> 4 & 1; @@ -3687,31 +3886,31 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_273; + float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_305; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_274; + float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_306; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_275; + float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_307; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_276; + float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_308; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_277; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_309; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_278; + float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_310; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_279; - float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_280; - float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_281; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_311; + float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_312; + float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_313; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); mbarrier_wait(p1b_ready_addr, 0); @@ -3770,24 +3969,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; - float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16) + dgf_16; + float qe_15 = q_t0_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15)) + dgf_16; int goff_16 = t_16 * rs_g; int qoff_16 = t_16 * qrs; { - auto _gst_282 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_o_16; + auto _gst_314 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_o_16; } if (t_16 < clen) { { - auto _gst_283 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_283)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_315 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_315)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_315) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_284 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_316 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_17 = 16 + 8 * th + 1; @@ -3813,24 +4017,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; - float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17) + dgf_17; + float qe_16 = q_t0_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16)) + dgf_17; int goff_17 = t_17 * rs_g; int qoff_17 = t_17 * qrs; { - auto _gst_285 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast(_gst_285) + (0)) = dg_o_17; + auto _gst_317 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast(_gst_317) + (0)) = dg_o_17; } if (t_17 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_18 = 16 + 8 * th + 2; @@ -3856,24 +4065,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; - float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18) + dgf_18; + float qe_17 = q_t0_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17)) + dgf_18; int goff_18 = t_18 * rs_g; int qoff_18 = t_18 * qrs; { - auto _gst_288 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast(_gst_288) + (0)) = dg_o_18; + auto _gst_320 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = dg_o_18; } if (t_18 < clen) { { - auto _gst_289 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_321 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_290 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_322 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_19 = 16 + 8 * th + 3; @@ -3899,24 +4113,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; - float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19) + dgf_19; + float qe_18 = q_t0_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18)) + dgf_19; int goff_19 = t_19 * rs_g; int qoff_19 = t_19 * qrs; { - auto _gst_291 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast(_gst_291) + (0)) = dg_o_19; + auto _gst_323 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast(_gst_323) + (0)) = dg_o_19; } if (t_19 < clen) { { - auto _gst_292 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_324 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_293 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_325 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_20 = 16 + 8 * th + 4; @@ -3942,24 +4161,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; - float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20) + dgf_20; + float qe_19 = q_t0_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19)) + dgf_20; int goff_20 = t_20 * rs_g; int qoff_20 = t_20 * qrs; { - auto _gst_294 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast(_gst_294) + (0)) = dg_o_20; + auto _gst_326 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast(_gst_326) + (0)) = dg_o_20; } if (t_20 < clen) { { - auto _gst_295 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_327 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_296 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_328 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_21 = 16 + 8 * th + 5; @@ -3985,24 +4209,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; - float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21) + dgf_21; + float qe_20 = q_t0_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20)) + dgf_21; int goff_21 = t_21 * rs_g; int qoff_21 = t_21 * qrs; { - auto _gst_297 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast(_gst_297) + (0)) = dg_o_21; + auto _gst_329 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast(_gst_329) + (0)) = dg_o_21; } if (t_21 < clen) { { - auto _gst_298 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_330 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_299 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_331 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_22 = 16 + 8 * th + 6; @@ -4028,24 +4257,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; - float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22) + dgf_22; + float qe_21 = q_t0_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21)) + dgf_22; int goff_22 = t_22 * rs_g; int qoff_22 = t_22 * qrs; { - auto _gst_300 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_300)); - *(reinterpret_cast(_gst_300) + (0)) = dg_o_22; + auto _gst_332 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_332)); + *(reinterpret_cast(_gst_332) + (0)) = dg_o_22; } if (t_22 < clen) { { - auto _gst_301 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_301)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_333 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_333)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_302 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_302)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_302) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_334 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_334)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_23 = 16 + 8 * th + 7; @@ -4071,24 +4305,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; - float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23) + dgf_23; + float qe_22 = q_t0_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22)) + dgf_23; int goff_23 = t_23 * rs_g; int qoff_23 = t_23 * qrs; { - auto _gst_303 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_303)); - *(reinterpret_cast(_gst_303) + (0)) = dg_o_23; + auto _gst_335 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_335)); + *(reinterpret_cast(_gst_335) + (0)) = dg_o_23; } if (t_23 < clen) { { - auto _gst_304 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_304)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_336 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_336)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_336) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_305 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_305)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_305) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_337 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_337)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dk_o_23; } } int b4_1 = lane_1 >> 4 & 1; @@ -4096,31 +4335,31 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_306; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_338; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_307; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_339; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_308; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_340; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_309; + float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_341; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_310; + float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_342; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_311; + float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_343; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_312; - float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_313; - float _shfl_xor_sync_314 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_314; + float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_344; + float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_345; + float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_346; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); mbarrier_wait(epi_full_addr + (th) * 8, 0); @@ -4173,24 +4412,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; - float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24) + dgf_24; + float qe_23 = q_t0_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23)) + dgf_24; int goff_24 = t_24 * rs_g; int qoff_24 = t_24 * qrs; { - auto _gst_315 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_o_24; + auto _gst_347 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_o_24; } if (t_24 < clen) { { - auto _gst_316 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_348 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_348) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_317 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_349 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_349) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_25 = 8 * th + 1; @@ -4216,24 +4460,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_25 = dk2_25 * bt_t_25; float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; - float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25) + dgf_25; + float qe_24 = q_t0_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24)) + dgf_25; int goff_25 = t_25 * rs_g; int qoff_25 = t_25 * qrs; { - auto _gst_318 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = dg_o_25; + auto _gst_350 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_350)); + *(reinterpret_cast(_gst_350) + (0)) = dg_o_25; } if (t_25 < clen) { { - auto _gst_319 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_351 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_320 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_352 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_26 = 8 * th + 2; @@ -4259,24 +4508,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_26 = dk2_26 * bt_t_26; float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; - float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26) + dgf_26; + float qe_25 = q_t0_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25)) + dgf_26; int goff_26 = t_26 * rs_g; int qoff_26 = t_26 * qrs; { - auto _gst_321 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast(_gst_321) + (0)) = dg_o_26; + auto _gst_353 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast(_gst_353) + (0)) = dg_o_26; } if (t_26 < clen) { { - auto _gst_322 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_354 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_323 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_355 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_27 = 8 * th + 3; @@ -4302,24 +4556,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_27 = dk2_27 * bt_t_27; float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; - float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27) + dgf_27; + float qe_26 = q_t0_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26)) + dgf_27; int goff_27 = t_27 * rs_g; int qoff_27 = t_27 * qrs; { - auto _gst_324 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast(_gst_324) + (0)) = dg_o_27; + auto _gst_356 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast(_gst_356) + (0)) = dg_o_27; } if (t_27 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_28 = 8 * th + 4; @@ -4345,24 +4604,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_28 = dk2_28 * bt_t_28; float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; - float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28) + dgf_28; + float qe_27 = q_t0_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27)) + dgf_28; int goff_28 = t_28 * rs_g; int qoff_28 = t_28 * qrs; { - auto _gst_327 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast(_gst_327) + (0)) = dg_o_28; + auto _gst_359 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast(_gst_359) + (0)) = dg_o_28; } if (t_28 < clen) { { - auto _gst_328 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_360 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_329 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_361 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_29 = 8 * th + 5; @@ -4388,24 +4652,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_29 = dk2_29 * bt_t_29; float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; - float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29) + dgf_29; + float qe_28 = q_t0_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28)) + dgf_29; int goff_29 = t_29 * rs_g; int qoff_29 = t_29 * qrs; { - auto _gst_330 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast(_gst_330) + (0)) = dg_o_29; + auto _gst_362 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast(_gst_362) + (0)) = dg_o_29; } if (t_29 < clen) { { - auto _gst_331 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_363 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_332 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_364 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_30 = 8 * th + 6; @@ -4431,24 +4700,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_30 = dk2_30 * bt_t_30; float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; - float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30) + dgf_30; + float qe_29 = q_t0_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29)) + dgf_30; int goff_30 = t_30 * rs_g; int qoff_30 = t_30 * qrs; { - auto _gst_333 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast(_gst_333) + (0)) = dg_o_30; + auto _gst_365 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast(_gst_365) + (0)) = dg_o_30; } if (t_30 < clen) { { - auto _gst_334 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_366 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_335 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_335)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_335) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_367 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_367)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_367) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_31 = 8 * th + 7; @@ -4474,24 +4748,29 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d float dk2b_31 = dk2_31 * bt_t_31; float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; - float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31) + dgf_31; + float qe_30 = q_t0_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30)) + dgf_31; int goff_31 = t_31 * rs_g; int qoff_31 = t_31 * qrs; { - auto _gst_336 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_336)); - *(reinterpret_cast(_gst_336) + (0)) = dg_o_31; + auto _gst_368 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_368)); + *(reinterpret_cast(_gst_368) + (0)) = dg_o_31; } if (t_31 < clen) { { - auto _gst_337 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_337)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_369 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_369)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_369) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_338 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_338)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_338) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_370 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_370)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_370) + (0)) = (__nv_bfloat16)dk_o_31; } } int b4_2 = lane_1 >> 4 & 1; @@ -4499,31 +4778,31 @@ kernel_cake_kda_train_54786ffd01bd9b6bb416(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_371; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_372; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_341; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_373; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_342; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_374; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_343; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_375; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_344; + float _shfl_xor_sync_376 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_376; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_345; - float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_346; - float _shfl_xor_sync_347 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_347; + float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_377; + float _shfl_xor_sync_378 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_378; + float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_379; if ((lane_1 & 3) == 0) { int e_star = lane_1 >> 2 & 7; pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_2; diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu index 952c3625eee..9970f4baf13 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_aa6b5c23c97e1cb9880a_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2249,12 +2272,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_3 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * e2_3); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_3, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_3 : __fmaf_rn(dl, _rcp_170, e2_3)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_3, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2278,12 +2307,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_4 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * e2_4); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_4, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_4, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_4 : __fmaf_rn(dl_1, _rcp_174, e2_4)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_4, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2307,12 +2342,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_5 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * e2_5); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_5, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_5, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_5 : __fmaf_rn(dl_3, _rcp_178, e2_5)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_5, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2336,12 +2377,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_6 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * e2_6); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_6, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_6, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_6 : __fmaf_rn(dl_5, _rcp_182, e2_6)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_6, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_2 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_1 = approx_exp2(dc2_2.x); float e2_1_1 = approx_exp2(dc2_2.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_190 = approx_rcp(k_t0_2); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_191 = approx_rcp(k_t1_2); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_2.x; pan_641[3] = p2_2.y; int t0_3 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_2 = approx_exp2(dc2_3.x); float e2_1_2 = approx_exp2(dc2_3.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_194 = approx_rcp(k_t0_3); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_195 = approx_rcp(k_t1_3); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_3.x; pan_641[5] = p2_3.y; int t0_4 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(_f2_61, make_float2(g_t0_4, g_t1_4)); float e2_0_3 = approx_exp2(dc2_4.x); float e2_1_3 = approx_exp2(dc2_4.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_198 = approx_rcp(k_t0_4); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_199 = approx_rcp(k_t1_4); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_4.x; pan_641[7] = p2_4.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2505,12 +2576,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_202 = approx_rcp(k_t0_5); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_203 = approx_rcp(k_t1_5); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_5.x; pan_703[1] = p2_5.y; int t0_6 = 8 * th + 2; @@ -2534,12 +2611,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_2 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_2.x); float en_1_1 = approx_exp2(dn2_2.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_206 = approx_rcp(k_t0_6); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_207 = approx_rcp(k_t1_6); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_6.x; pan_703[3] = p2_6.y; int t0_7 = 8 * th + 4; @@ -2563,12 +2646,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_3 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_3.x); float en_1_2 = approx_exp2(dn2_3.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_210 = approx_rcp(k_t0_7); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_211 = approx_rcp(k_t1_7); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_7.x; pan_703[5] = p2_7.y; int t0_8 = 8 * th + 6; @@ -2592,12 +2681,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_4 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_4.x); float en_1_3 = approx_exp2(dn2_4.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_214 = approx_rcp(k_t0_8); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_215 = approx_rcp(k_t1_8); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_8.x; pan_703[7] = p2_8.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2639,12 +2734,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_5 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_5.x); float en_1_4 = approx_exp2(dn2_5.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_218 = approx_rcp(k_t0_9); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_219 = approx_rcp(k_t1_9); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_9.x; pan_777[1] = p2_9.y; int t0_10 = 16 + 8 * th + 2; @@ -2668,12 +2769,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_6 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_6.x); float en_1_5 = approx_exp2(dn2_6.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_222 = approx_rcp(k_t0_10); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_223 = approx_rcp(k_t1_10); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_10.x; pan_777[3] = p2_10.y; int t0_11 = 16 + 8 * th + 4; @@ -2697,12 +2804,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_7 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_7.x); float en_1_6 = approx_exp2(dn2_7.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_226 = approx_rcp(k_t0_11); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_227 = approx_rcp(k_t1_11); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_11.x; pan_777[5] = p2_11.y; int t0_12 = 16 + 8 * th + 6; @@ -2726,12 +2839,18 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 dn2_8 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_8.x); float en_1_7 = approx_exp2(dn2_8.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_230 = approx_rcp(k_t0_12); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_231 = approx_rcp(k_t1_12); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_12.x; pan_777[7] = p2_12.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2802,37 +2921,37 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_4 = approx_exp2(d2_57.x); float e1_4 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_4, e1_4); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_4, e1_4); float2 v2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_4.x; v_562_0[1] = v2_4.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_5 = approx_exp2(d2_58.x); float e1_5 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_5, e1_5); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_5, e1_5); float2 v2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_5.x; v_562_0[3] = v2_5.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_6 = approx_exp2(d2_59.x); float e1_6 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_6, e1_6); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_6, e1_6); float2 v2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_6.x; v_562_0[5] = v2_6.y; float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_7 = approx_exp2(d2_60.x); float e1_7 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_7, e1_7); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_7, e1_7); float2 v2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_7.x; v_562_0[7] = v2_7.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2966,7 +3085,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_0 = dk2_1_0 * bt_t_1_0; float dq_o_1_0 = dq2_1_0 + dqf_1_0; float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; + float qe = q_t_2_0 * e_dg_1_0; + float kp = k_t_1_0 * bt_t_1_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_1_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, _tmem_load_9[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_1_0) * _tmem_load_10[7]); + float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, __fmaf_rn(dk2b_1_0 - _tmem_load_11[7], k_t_1_0, cr)) + dgf_1_0; s_run[0] = s_run[0] + dg_o_1_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -2976,14 +3100,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_0 = t_1_0 * qrs; if (t_1_0 < clen) { { - auto _gst_216 = dq_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_1_0; + auto _gst_248 = dq_base + qoff_1_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_1_0; } { - auto _gst_217 = dk_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_1_0; + auto _gst_249 = dk_base + qoff_1_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_1_0; } } int t_1_1 = 32 + 8 * th + 6; @@ -3009,7 +3133,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_1 = dk2_1_1 * bt_t_1_1; float dq_o_1_1 = dq2_1_1 + dqf_1_1; float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; + float qe_0 = q_t_2_1 * e_dg_1_1; + float kp_12 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_9[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12 * e_dg_1_1) * _tmem_load_10[6]); + float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_11[6], k_t_1_1, cr_0)) + dgf_1_1; s_run[0] = s_run[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3019,14 +3148,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_1 = t_1_1 * qrs; if (t_1_1 < clen) { { - auto _gst_218 = dq_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1_1; + auto _gst_250 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1_1; } { - auto _gst_219 = dk_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1_1; + auto _gst_251 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1_1; } } int t_1_2 = 32 + 8 * th + 5; @@ -3052,7 +3181,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_2 = dk2_1_2 * bt_t_1_2; float dq_o_1_2 = dq2_1_2 + dqf_1_2; float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; + float qe_1 = q_t_2_2 * e_dg_1_2; + float kp_13 = k_t_1_2 * bt_t_1_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_13; + float kbe_1 = (float)kb_1 * e_dg_1_2; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, _tmem_load_9[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_13 * e_dg_1_2) * _tmem_load_10[5]); + float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, __fmaf_rn(dk2b_1_2 - _tmem_load_11[5], k_t_1_2, cr_2)) + dgf_1_2; s_run[0] = s_run[0] + dg_o_1_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3062,14 +3196,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_2 = t_1_2 * qrs; if (t_1_2 < clen) { { - auto _gst_220 = dq_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1_2; + auto _gst_252 = dq_base + qoff_1_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_1_2; } { - auto _gst_221 = dk_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1_2; + auto _gst_253 = dk_base + qoff_1_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_1_2; } } int t_1_3 = 32 + 8 * th + 4; @@ -3095,7 +3229,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_3 = dk2_1_3 * bt_t_1_3; float dq_o_1_3 = dq2_1_3 + dqf_1_3; float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; + float qe_2 = q_t_2_3 * e_dg_1_3; + float kp_14 = k_t_1_3 * bt_t_1_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_1_3; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_9[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_1_3) * _tmem_load_10[4]); + float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, __fmaf_rn(dk2b_1_3 - _tmem_load_11[4], k_t_1_3, cr_3)) + dgf_1_3; s_run[0] = s_run[0] + dg_o_1_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3105,14 +3244,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_3 = t_1_3 * qrs; if (t_1_3 < clen) { { - auto _gst_222 = dq_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_1_3; + auto _gst_254 = dq_base + qoff_1_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_1_3; } { - auto _gst_223 = dk_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_1_3; + auto _gst_255 = dk_base + qoff_1_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_1_3; } } int t_1_4 = 32 + 8 * th + 3; @@ -3138,7 +3277,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_4 = dk2_1_4 * bt_t_1_4; float dq_o_1_4 = dq2_1_4 + dqf_1_4; float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; + float qe_3 = q_t_2_4 * e_dg_1_4; + float kp_15 = k_t_1_4 * bt_t_1_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_1_4; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_9[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_1_4) * _tmem_load_10[3]); + float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, __fmaf_rn(dk2b_1_4 - _tmem_load_11[3], k_t_1_4, cr_4)) + dgf_1_4; s_run[0] = s_run[0] + dg_o_1_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3148,14 +3292,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_4 = t_1_4 * qrs; if (t_1_4 < clen) { { - auto _gst_224 = dq_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_1_4; + auto _gst_256 = dq_base + qoff_1_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_1_4; } { - auto _gst_225 = dk_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_1_4; + auto _gst_257 = dk_base + qoff_1_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_1_4; } } int t_1_5 = 32 + 8 * th + 2; @@ -3181,7 +3325,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_5 = dk2_1_5 * bt_t_1_5; float dq_o_1_5 = dq2_1_5 + dqf_1_5; float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; + float qe_4 = q_t_2_5 * e_dg_1_5; + float kp_16 = k_t_1_5 * bt_t_1_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_1_5; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_9[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_1_5) * _tmem_load_10[2]); + float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, __fmaf_rn(dk2b_1_5 - _tmem_load_11[2], k_t_1_5, cr_5)) + dgf_1_5; s_run[0] = s_run[0] + dg_o_1_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3191,14 +3340,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_5 = t_1_5 * qrs; if (t_1_5 < clen) { { - auto _gst_226 = dq_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_1_5; + auto _gst_258 = dq_base + qoff_1_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_1_5; } { - auto _gst_227 = dk_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_1_5; + auto _gst_259 = dk_base + qoff_1_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_1_5; } } int t_1_6 = 32 + 8 * th + 1; @@ -3224,7 +3373,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_6 = dk2_1_6 * bt_t_1_6; float dq_o_1_6 = dq2_1_6 + dqf_1_6; float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; + float qe_5 = q_t_2_6 * e_dg_1_6; + float kp_17 = k_t_1_6 * bt_t_1_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_1_6; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_9[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_1_6) * _tmem_load_10[1]); + float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, __fmaf_rn(dk2b_1_6 - _tmem_load_11[1], k_t_1_6, cr_6)) + dgf_1_6; s_run[0] = s_run[0] + dg_o_1_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3234,14 +3388,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_6 = t_1_6 * qrs; if (t_1_6 < clen) { { - auto _gst_228 = dq_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_1_6; + auto _gst_260 = dq_base + qoff_1_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_1_6; } { - auto _gst_229 = dk_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_1_6; + auto _gst_261 = dk_base + qoff_1_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_1_6; } } int t_1_7 = 32 + 8 * th; @@ -3267,7 +3421,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_1_7 = dk2_1_7 * bt_t_1_7; float dq_o_1_7 = dq2_1_7 + dqf_1_7; float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; + float qe_6 = q_t_2_7 * e_dg_1_7; + float kp_18 = k_t_1_7 * bt_t_1_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_1_7; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_9[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_1_7) * _tmem_load_10[0]); + float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, __fmaf_rn(dk2b_1_7 - _tmem_load_11[0], k_t_1_7, cr_7)) + dgf_1_7; s_run[0] = s_run[0] + dg_o_1_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3277,14 +3436,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_1_7 = t_1_7 * qrs; if (t_1_7 < clen) { { - auto _gst_230 = dq_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_1_7; + auto _gst_262 = dq_base + qoff_1_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_1_7; } { - auto _gst_231 = dk_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_1_7; + auto _gst_263 = dk_base + qoff_1_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_1_7; } } float tot = s_run[0]; @@ -3293,31 +3452,31 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2_1044[2]; float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); + v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); + v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); + float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3377,7 +3536,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; + float qe_7 = q_t_1_8 * e_dg_0; + float kp_19 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_0; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_14[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_0) * _tmem_load_15[7]); + float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_16[7], k_t_0, cr_8)) + dgf_0; s_run_1059[0] = s_run_1059[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3387,14 +3551,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_241 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_273 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_242 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_274 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_2 = 48 + 8 * th + 6; @@ -3420,7 +3584,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[6]; - float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, (dk2b_2 - _tmem_load_16[6]) * k_t_2) + dgf_2; + float qe_8 = q_t_1_9 * e_dg_2; + float kp_20 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_2; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_14[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_2) * _tmem_load_15[6]); + float dg_o_2 = __fmaf_rn(q_t_1_9, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_16[6], k_t_2, cr_9)) + dgf_2; s_run_1059[0] = s_run_1059[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3430,14 +3599,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_243 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_275 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_244 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_276 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 48 + 8 * th + 5; @@ -3463,7 +3632,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[5]; - float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, (dk2b_3 - _tmem_load_16[5]) * k_t_3) + dgf_3; + float qe_9 = q_t_1_10 * e_dg_3; + float kp_21 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_3; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_14[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_3) * _tmem_load_15[5]); + float dg_o_3 = __fmaf_rn(q_t_1_10, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_16[5], k_t_3, cr_10)) + dgf_3; s_run_1059[0] = s_run_1059[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3473,14 +3647,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_245 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_277 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_246 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_278 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 48 + 8 * th + 4; @@ -3506,7 +3680,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[4]; - float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, (dk2b_4 - _tmem_load_16[4]) * k_t_4) + dgf_4; + float qe_10 = q_t_1_11 * e_dg_4; + float kp_22 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_4; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_14[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_4) * _tmem_load_15[4]); + float dg_o_4 = __fmaf_rn(q_t_1_11, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_16[4], k_t_4, cr_11)) + dgf_4; s_run_1059[0] = s_run_1059[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3516,14 +3695,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_247 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_279 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_248 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_280 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 48 + 8 * th + 3; @@ -3549,7 +3728,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[3]; - float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, (dk2b_5 - _tmem_load_16[3]) * k_t_5) + dgf_5; + float qe_11 = q_t_1_12 * e_dg_5; + float kp_23 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_5; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_14[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_5) * _tmem_load_15[3]); + float dg_o_5 = __fmaf_rn(q_t_1_12, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_16[3], k_t_5, cr_12)) + dgf_5; s_run_1059[0] = s_run_1059[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3559,14 +3743,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_249 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_281 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_250 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_282 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 48 + 8 * th + 2; @@ -3592,7 +3776,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[2]; - float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, (dk2b_6 - _tmem_load_16[2]) * k_t_6) + dgf_6; + float qe_12 = q_t_1_13 * e_dg_6; + float kp_24 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_6; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_14[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_6) * _tmem_load_15[2]); + float dg_o_6 = __fmaf_rn(q_t_1_13, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_16[2], k_t_6, cr_13)) + dgf_6; s_run_1059[0] = s_run_1059[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3602,14 +3791,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_251 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_283 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_252 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_284 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 48 + 8 * th + 1; @@ -3635,7 +3824,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[1]; - float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, (dk2b_7 - _tmem_load_16[1]) * k_t_7) + dgf_7; + float qe_13 = q_t_1_14 * e_dg_7; + float kp_25 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_7; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_14[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_7) * _tmem_load_15[1]); + float dg_o_7 = __fmaf_rn(q_t_1_14, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_16[1], k_t_7, cr_14)) + dgf_7; s_run_1059[0] = s_run_1059[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3645,14 +3839,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_253 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_285 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_254 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_286 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_7; } } int t_8 = 48 + 8 * th; @@ -3678,7 +3872,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_16[0]; - float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, (dk2b_8 - _tmem_load_16[0]) * k_t_8) + dgf_8; + float qe_14 = q_t_1_15 * e_dg_8; + float kp_26 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_8; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_14[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_8) * _tmem_load_15[0]); + float dg_o_8 = __fmaf_rn(q_t_1_15, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_16[0], k_t_8, cr_15)) + dgf_8; s_run_1059[0] = s_run_1059[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3688,14 +3887,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_255 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_287 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_256 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_288 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_8; } } float tot_1228 = s_run_1059[0]; @@ -3704,36 +3903,36 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int b2_1231 = lane_1 >> 2 & 1; float v4_1232[4]; float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_289; float send_2 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_290; float send_3 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_291; float send_4 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_292; float v2_1237[2]; float send_5 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_293; float send_6 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_6, 8); + v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_294; float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); + float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); + v1_1241 = v1_1241 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); + v1_1241 = v1_1241 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_1228) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_1228) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3762,9 +3961,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_0 = offset + _tmem_load_17[0]; if (t_9 < clen) { { - auto _gst_267 = dg_base + goff_1_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_1_0; + auto _gst_299 = dg_base + goff_1_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_1_0; } } int t_10 = 48 + 8 * th + 1; @@ -3772,9 +3971,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_1 = offset + _tmem_load_17[1]; if (t_10 < clen) { { - auto _gst_268 = dg_base + goff_1_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1_1; + auto _gst_300 = dg_base + goff_1_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1_1; } } int t_11 = 48 + 8 * th + 2; @@ -3782,9 +3981,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_2 = offset + _tmem_load_17[2]; if (t_11 < clen) { { - auto _gst_269 = dg_base + goff_1_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_1_2; + auto _gst_301 = dg_base + goff_1_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_1_2; } } int t_12 = 48 + 8 * th + 3; @@ -3792,9 +3991,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_3 = offset + _tmem_load_17[3]; if (t_12 < clen) { { - auto _gst_270 = dg_base + goff_1_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_1_3; + auto _gst_302 = dg_base + goff_1_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_1_3; } } int t_13 = 48 + 8 * th + 4; @@ -3802,9 +4001,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_4 = offset + _tmem_load_17[4]; if (t_13 < clen) { { - auto _gst_271 = dg_base + goff_1_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_1_4; + auto _gst_303 = dg_base + goff_1_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_1_4; } } int t_14 = 48 + 8 * th + 5; @@ -3812,9 +4011,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_5 = offset + _tmem_load_17[5]; if (t_14 < clen) { { - auto _gst_272 = dg_base + goff_1_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_1_5; + auto _gst_304 = dg_base + goff_1_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_1_5; } } int t_15 = 48 + 8 * th + 6; @@ -3822,9 +4021,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_6 = offset + _tmem_load_17[6]; if (t_15 < clen) { { - auto _gst_273 = dg_base + goff_1_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_1_6; + auto _gst_305 = dg_base + goff_1_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_1_6; } } int t_16 = 48 + 8 * th + 7; @@ -3832,9 +4031,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1_7 = offset + _tmem_load_17[7]; if (t_16 < clen) { { - auto _gst_274 = dg_base + goff_1_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_1_7; + auto _gst_306 = dg_base + goff_1_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_1_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3852,9 +4051,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_0 = offset_1250 + _tmem_load_19[0]; if (t_17 < clen) { { - auto _gst_275 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_0; + auto _gst_307 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_0; } } int t_18 = 32 + 8 * th + 1; @@ -3862,9 +4061,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_1 = offset_1250 + _tmem_load_19[1]; if (t_18 < clen) { { - auto _gst_276 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_1; + auto _gst_308 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_1; } } int t_19 = 32 + 8 * th + 2; @@ -3872,9 +4071,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_2 = offset_1250 + _tmem_load_19[2]; if (t_19 < clen) { { - auto _gst_277 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_2; + auto _gst_309 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_2; } } int t_20 = 32 + 8 * th + 3; @@ -3882,9 +4081,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_3 = offset_1250 + _tmem_load_19[3]; if (t_20 < clen) { { - auto _gst_278 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_3; + auto _gst_310 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_3; } } int t_21 = 32 + 8 * th + 4; @@ -3892,9 +4091,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_4 = offset_1250 + _tmem_load_19[4]; if (t_21 < clen) { { - auto _gst_279 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_4; + auto _gst_311 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_4; } } int t_22 = 32 + 8 * th + 5; @@ -3902,9 +4101,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_5 = offset_1250 + _tmem_load_19[5]; if (t_22 < clen) { { - auto _gst_280 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_5; + auto _gst_312 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_5; } } int t_23 = 32 + 8 * th + 6; @@ -3912,9 +4111,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_6 = offset_1250 + _tmem_load_19[6]; if (t_23 < clen) { { - auto _gst_281 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_6; + auto _gst_313 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_6; } } int t_24 = 32 + 8 * th + 7; @@ -3922,15 +4121,15 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_7 = offset_1250 + _tmem_load_19[7]; if (t_24 < clen) { { - auto _gst_282 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_7; + auto _gst_314 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi_1248 + t_lo_1249; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3990,7 +4189,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[7]; - float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, (dk2b_9 - _tmem_load_24[7]) * k_t_9) + dgf_9; + float qe_15 = q_t_1_16 * e_dg_9; + float kp_27 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_9; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_9) * _tmem_load_23[7]); + float dg_o_9 = __fmaf_rn(q_t_1_16, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_24[7], k_t_9, cr_16)) + dgf_9; s_run_1302[0] = s_run_1302[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4000,14 +4204,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_9 = t_25 * qrs; if (t_25 < clen) { { - auto _gst_284 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_316 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_285 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_317 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_26 = 16 + 8 * th + 6; @@ -4033,7 +4237,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[6]; - float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, (dk2b_10 - _tmem_load_24[6]) * k_t_10) + dgf_10; + float qe_16 = q_t_1_17 * e_dg_10; + float kp_28 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_10; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_10) * _tmem_load_23[6]); + float dg_o_10 = __fmaf_rn(q_t_1_17, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_24[6], k_t_10, cr_17)) + dgf_10; s_run_1302[0] = s_run_1302[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4043,14 +4252,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_10 = t_26 * qrs; if (t_26 < clen) { { - auto _gst_286 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_318 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_287 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_319 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_27 = 16 + 8 * th + 5; @@ -4076,7 +4285,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[5]; - float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, (dk2b_11 - _tmem_load_24[5]) * k_t_11) + dgf_11; + float qe_17 = q_t_1_18 * e_dg_11; + float kp_29 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_11; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_11) * _tmem_load_23[5]); + float dg_o_11 = __fmaf_rn(q_t_1_18, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_24[5], k_t_11, cr_18)) + dgf_11; s_run_1302[0] = s_run_1302[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4086,14 +4300,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_11 = t_27 * qrs; if (t_27 < clen) { { - auto _gst_288 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_320 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_289 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_321 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_28 = 16 + 8 * th + 4; @@ -4119,7 +4333,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[4]; - float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, (dk2b_12 - _tmem_load_24[4]) * k_t_12) + dgf_12; + float qe_18 = q_t_1_19 * e_dg_12; + float kp_30 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_12; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_12) * _tmem_load_23[4]); + float dg_o_12 = __fmaf_rn(q_t_1_19, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_24[4], k_t_12, cr_19)) + dgf_12; s_run_1302[0] = s_run_1302[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4129,14 +4348,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_12 = t_28 * qrs; if (t_28 < clen) { { - auto _gst_290 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_322 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_291 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_323 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_29 = 16 + 8 * th + 3; @@ -4162,7 +4381,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[3]; - float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, (dk2b_13 - _tmem_load_24[3]) * k_t_13) + dgf_13; + float qe_19 = q_t_1_20 * e_dg_13; + float kp_31 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_13; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_13) * _tmem_load_23[3]); + float dg_o_13 = __fmaf_rn(q_t_1_20, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_24[3], k_t_13, cr_20)) + dgf_13; s_run_1302[0] = s_run_1302[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4172,14 +4396,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_13 = t_29 * qrs; if (t_29 < clen) { { - auto _gst_292 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_324 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_293 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_325 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_30 = 16 + 8 * th + 2; @@ -4205,7 +4429,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[2]; - float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, (dk2b_14 - _tmem_load_24[2]) * k_t_14) + dgf_14; + float qe_20 = q_t_1_21 * e_dg_14; + float kp_32 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_14; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_14) * _tmem_load_23[2]); + float dg_o_14 = __fmaf_rn(q_t_1_21, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_24[2], k_t_14, cr_21)) + dgf_14; s_run_1302[0] = s_run_1302[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4215,14 +4444,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_14 = t_30 * qrs; if (t_30 < clen) { { - auto _gst_294 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_326 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_295 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_327 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_31 = 16 + 8 * th + 1; @@ -4248,7 +4477,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[1]; - float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, (dk2b_15 - _tmem_load_24[1]) * k_t_15) + dgf_15; + float qe_21 = q_t_1_22 * e_dg_15; + float kp_33 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_15; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_15) * _tmem_load_23[1]); + float dg_o_15 = __fmaf_rn(q_t_1_22, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_24[1], k_t_15, cr_22)) + dgf_15; s_run_1302[0] = s_run_1302[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4258,14 +4492,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_15 = t_31 * qrs; if (t_31 < clen) { { - auto _gst_296 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_328 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_297 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_329 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_15; } } int t_32 = 16 + 8 * th; @@ -4291,7 +4525,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_24[0]; - float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, (dk2b_16 - _tmem_load_24[0]) * k_t_16) + dgf_16; + float qe_22 = q_t_1_23 * e_dg_16; + float kp_34 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_16; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_16) * _tmem_load_23[0]); + float dg_o_16 = __fmaf_rn(q_t_1_23, dq2_16, __fmaf_rn(dk2b_16 - _tmem_load_24[0], k_t_16, cr_23)) + dgf_16; s_run_1302[0] = s_run_1302[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4301,14 +4540,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_298 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_330 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_299 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_331 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_16; } } float tot_1471 = s_run_1302[0]; @@ -4317,37 +4556,37 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; float send_7 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_332; float send_8 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_333; float send_9 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_334; float send_10 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_335; float v2_1480[2]; float send_11 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_336; float send_12 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_337; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1471) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4376,9 +4615,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_8 = offset_1488 + _tmem_load_25[0]; if (t_33 < clen) { { - auto _gst_310 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_8; + auto _gst_342 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_8; } } int t_34 = 16 + 8 * th + 1; @@ -4386,9 +4625,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_9 = offset_1488 + _tmem_load_25[1]; if (t_34 < clen) { { - auto _gst_311 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_9; + auto _gst_343 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_9; } } int t_35 = 16 + 8 * th + 2; @@ -4396,9 +4635,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_10 = offset_1488 + _tmem_load_25[2]; if (t_35 < clen) { { - auto _gst_312 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_10; + auto _gst_344 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_10; } } int t_36 = 16 + 8 * th + 3; @@ -4406,9 +4645,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_11 = offset_1488 + _tmem_load_25[3]; if (t_36 < clen) { { - auto _gst_313 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_11; + auto _gst_345 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_11; } } int t_37 = 16 + 8 * th + 4; @@ -4416,9 +4655,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_12 = offset_1488 + _tmem_load_25[4]; if (t_37 < clen) { { - auto _gst_314 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_12; + auto _gst_346 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_12; } } int t_38 = 16 + 8 * th + 5; @@ -4426,9 +4665,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_13 = offset_1488 + _tmem_load_25[5]; if (t_38 < clen) { { - auto _gst_315 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_13; + auto _gst_347 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_13; } } int t_39 = 16 + 8 * th + 6; @@ -4436,9 +4675,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_14 = offset_1488 + _tmem_load_25[6]; if (t_39 < clen) { { - auto _gst_316 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_14; + auto _gst_348 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_14; } } int t_40 = 16 + 8 * th + 7; @@ -4446,15 +4685,15 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_15 = offset_1488 + _tmem_load_25[7]; if (t_40 < clen) { { - auto _gst_317 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_15; + auto _gst_349 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_1486 + t_lo_1487; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4508,7 +4747,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[7]; - float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, (dk2b_17 - _tmem_load_28[7]) * k_t_17) + dgf_17; + float qe_23 = q_t_1_24 * e_dg_17; + float kp_35 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_35; + float kbe_23 = (float)kb_23 * e_dg_17; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_35 * e_dg_17) * _tmem_load_27[7]); + float dg_o_17 = __fmaf_rn(q_t_1_24, dq2_17, __fmaf_rn(dk2b_17 - _tmem_load_28[7], k_t_17, cr_24)) + dgf_17; s_run_1524[0] = s_run_1524[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4518,14 +4762,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_17 = t_41 * qrs; if (t_41 < clen) { { - auto _gst_319 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_351 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_320 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_352 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_42 = 8 * th + 6; @@ -4551,7 +4795,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[6]; - float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, (dk2b_18 - _tmem_load_28[6]) * k_t_18) + dgf_18; + float qe_24 = q_t_1_25 * e_dg_18; + float kp_36 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_36; + float kbe_24 = (float)kb_24 * e_dg_18; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_36 * e_dg_18) * _tmem_load_27[6]); + float dg_o_18 = __fmaf_rn(q_t_1_25, dq2_18, __fmaf_rn(dk2b_18 - _tmem_load_28[6], k_t_18, cr_25)) + dgf_18; s_run_1524[0] = s_run_1524[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4561,14 +4810,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_18 = t_42 * qrs; if (t_42 < clen) { { - auto _gst_321 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_353 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_322 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_354 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_43 = 8 * th + 5; @@ -4594,7 +4843,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[5]; - float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, (dk2b_19 - _tmem_load_28[5]) * k_t_19) + dgf_19; + float qe_25 = q_t_1_26 * e_dg_19; + float kp_37 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_37; + float kbe_25 = (float)kb_25 * e_dg_19; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_37 * e_dg_19) * _tmem_load_27[5]); + float dg_o_19 = __fmaf_rn(q_t_1_26, dq2_19, __fmaf_rn(dk2b_19 - _tmem_load_28[5], k_t_19, cr_26)) + dgf_19; s_run_1524[0] = s_run_1524[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4604,14 +4858,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_19 = t_43 * qrs; if (t_43 < clen) { { - auto _gst_323 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_355 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_324 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_356 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_44 = 8 * th + 4; @@ -4637,7 +4891,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[4]; - float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, (dk2b_20 - _tmem_load_28[4]) * k_t_20) + dgf_20; + float qe_26 = q_t_1_27 * e_dg_20; + float kp_38 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_38; + float kbe_26 = (float)kb_26 * e_dg_20; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_38 * e_dg_20) * _tmem_load_27[4]); + float dg_o_20 = __fmaf_rn(q_t_1_27, dq2_20, __fmaf_rn(dk2b_20 - _tmem_load_28[4], k_t_20, cr_27)) + dgf_20; s_run_1524[0] = s_run_1524[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4647,14 +4906,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_20 = t_44 * qrs; if (t_44 < clen) { { - auto _gst_325 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_357 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_326 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_358 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_45 = 8 * th + 3; @@ -4680,7 +4939,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[3]; - float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, (dk2b_21 - _tmem_load_28[3]) * k_t_21) + dgf_21; + float qe_27 = q_t_1_28 * e_dg_21; + float kp_39 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_39; + float kbe_27 = (float)kb_27 * e_dg_21; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_39 * e_dg_21) * _tmem_load_27[3]); + float dg_o_21 = __fmaf_rn(q_t_1_28, dq2_21, __fmaf_rn(dk2b_21 - _tmem_load_28[3], k_t_21, cr_28)) + dgf_21; s_run_1524[0] = s_run_1524[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4690,14 +4954,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_21 = t_45 * qrs; if (t_45 < clen) { { - auto _gst_327 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_359 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_328 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_360 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_46 = 8 * th + 2; @@ -4723,7 +4987,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[2]; - float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, (dk2b_22 - _tmem_load_28[2]) * k_t_22) + dgf_22; + float qe_28 = q_t_1_29 * e_dg_22; + float kp_40 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_40; + float kbe_28 = (float)kb_28 * e_dg_22; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_40 * e_dg_22) * _tmem_load_27[2]); + float dg_o_22 = __fmaf_rn(q_t_1_29, dq2_22, __fmaf_rn(dk2b_22 - _tmem_load_28[2], k_t_22, cr_29)) + dgf_22; s_run_1524[0] = s_run_1524[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4733,14 +5002,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_22 = t_46 * qrs; if (t_46 < clen) { { - auto _gst_329 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_361 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_330 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_362 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_47 = 8 * th + 1; @@ -4766,7 +5035,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[1]; - float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, (dk2b_23 - _tmem_load_28[1]) * k_t_23) + dgf_23; + float qe_29 = q_t_1_30 * e_dg_23; + float kp_41 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_41; + float kbe_29 = (float)kb_29 * e_dg_23; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_41 * e_dg_23) * _tmem_load_27[1]); + float dg_o_23 = __fmaf_rn(q_t_1_30, dq2_23, __fmaf_rn(dk2b_23 - _tmem_load_28[1], k_t_23, cr_30)) + dgf_23; s_run_1524[0] = s_run_1524[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4776,14 +5050,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_23 = t_47 * qrs; if (t_47 < clen) { { - auto _gst_331 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_363 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_332 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_364 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_23; } } int t_48 = 8 * th; @@ -4809,7 +5083,12 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + _tmem_load_28[0]; - float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, (dk2b_24 - _tmem_load_28[0]) * k_t_24) + dgf_24; + float qe_30 = q_t_1_31 * e_dg_24; + float kp_42 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_42; + float kbe_30 = (float)kb_30 * e_dg_24; + float cr_31 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_42 * e_dg_24) * _tmem_load_27[0]); + float dg_o_24 = __fmaf_rn(q_t_1_31, dq2_24, __fmaf_rn(dk2b_24 - _tmem_load_28[0], k_t_24, cr_31)) + dgf_24; s_run_1524[0] = s_run_1524[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4819,14 +5098,14 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_333 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_365 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_334 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_366 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_24; } } float tot_1693 = s_run_1524[0]; @@ -4835,35 +5114,35 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; float send_13 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_367; float send_14 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_368; float send_15 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_369; float send_16 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_370; float v2_1702[2]; float send_17 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_371; float send_18 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_372; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_1693) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4892,9 +5171,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_16 = offset_1710 + _tmem_load_29[0]; if (t_49 < clen) { { - auto _gst_345 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_16; + auto _gst_377 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_16; } } int t_50 = 8 * th + 1; @@ -4902,9 +5181,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_17 = offset_1710 + _tmem_load_29[1]; if (t_50 < clen) { { - auto _gst_346 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_17; + auto _gst_378 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_17; } } int t_51 = 8 * th + 2; @@ -4912,9 +5191,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_18 = offset_1710 + _tmem_load_29[2]; if (t_51 < clen) { { - auto _gst_347 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_18; + auto _gst_379 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_18; } } int t_52 = 8 * th + 3; @@ -4922,9 +5201,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_19 = offset_1710 + _tmem_load_29[3]; if (t_52 < clen) { { - auto _gst_348 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_19; + auto _gst_380 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_19; } } int t_53 = 8 * th + 4; @@ -4932,9 +5211,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_20 = offset_1710 + _tmem_load_29[4]; if (t_53 < clen) { { - auto _gst_349 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_20; + auto _gst_381 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_20; } } int t_54 = 8 * th + 5; @@ -4942,9 +5221,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_21 = offset_1710 + _tmem_load_29[5]; if (t_54 < clen) { { - auto _gst_350 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_21; + auto _gst_382 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_21; } } int t_55 = 8 * th + 6; @@ -4952,9 +5231,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_22 = offset_1710 + _tmem_load_29[6]; if (t_55 < clen) { { - auto _gst_351 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_22; + auto _gst_383 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_22; } } int t_56 = 8 * th + 7; @@ -4962,9 +5241,9 @@ kernel_cake_kda_train_aa6b5c23c97e1cb9880a(const __grid_constant__ CUtensorMap d float dg_23 = offset_1710 + _tmem_load_29[7]; if (t_56 < clen) { { - auto _gst_352 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_23; + auto _gst_384 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1708 + t_lo_1709; diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu index ecc38ee65b5..7082e24aea8 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_c27b09eb5f5881be2969_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2246,15 +2269,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2275,15 +2304,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2304,15 +2339,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2333,15 +2374,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2502,15 +2573,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2531,15 +2608,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2560,15 +2643,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2589,15 +2678,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2636,15 +2731,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2665,15 +2766,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2694,15 +2801,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2723,15 +2836,21 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2802,37 +2921,37 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2965,19 +3084,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_0 = dk2_0 * bt_t_0; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; - float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0) + dgf_0; + float qe = q_t0_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dg_o_0 = __fmaf_rn(q_t0_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr)) + dgf_0; dgo[0] = dg_o_0; int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_248 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_249 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -3003,19 +3127,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_1 = dk2_1 * bt_t_1; float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; - float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1) + dgf_1; + float qe_0 = q_t0_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dg_o_1 = __fmaf_rn(q_t0_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0)) + dgf_1; dgo[1] = dg_o_1; int qoff_1 = t_1 * qrs; if (t_1 < clen) { { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_250 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_251 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3041,19 +3170,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_2 = dk2_2 * bt_t_2; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; - float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2) + dgf_2; + float qe_1 = q_t0_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dg_o_2 = __fmaf_rn(q_t0_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1)) + dgf_2; dgo[2] = dg_o_2; int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_252 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_253 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3079,19 +3213,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_3 = dk2_3 * bt_t_3; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; - float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3) + dgf_3; + float qe_2 = q_t0_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dg_o_3 = __fmaf_rn(q_t0_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2)) + dgf_3; dgo[3] = dg_o_3; int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_254 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_255 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3117,19 +3256,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_4 = dk2_4 * bt_t_4; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; - float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4) + dgf_4; + float qe_3 = q_t0_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dg_o_4 = __fmaf_rn(q_t0_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3)) + dgf_4; dgo[4] = dg_o_4; int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_256 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_257 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3155,19 +3299,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_5 = dk2_5 * bt_t_5; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; - float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5) + dgf_5; + float qe_4 = q_t0_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dg_o_5 = __fmaf_rn(q_t0_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4)) + dgf_5; dgo[5] = dg_o_5; int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_258 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_259 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3193,19 +3342,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_6 = dk2_6 * bt_t_6; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; - float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6) + dgf_6; + float qe_5 = q_t0_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dg_o_6 = __fmaf_rn(q_t0_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5)) + dgf_6; dgo[6] = dg_o_6; int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_260 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_261 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3231,19 +3385,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_7 = dk2_7 * bt_t_7; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; - float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7) + dgf_7; + float qe_6 = q_t0_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dg_o_7 = __fmaf_rn(q_t0_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6)) + dgf_7; dgo[7] = dg_o_7; int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_262 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_263 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_7; } } float s_run[1]; @@ -3272,31 +3431,31 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3355,19 +3514,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_8 = dk2_8 * bt_t_8; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; - float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8) + dgf_8; + float qe_7 = q_t0_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dg_o_8 = __fmaf_rn(q_t0_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7)) + dgf_8; dgo_0[0] = dg_o_8; int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_273 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_274 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3393,19 +3557,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_9 = dk2_9 * bt_t_9; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; - float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9) + dgf_9; + float qe_8 = q_t0_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dg_o_9 = __fmaf_rn(q_t0_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8)) + dgf_9; dgo_0[1] = dg_o_9; int qoff_9 = t_9 * qrs; if (t_9 < clen) { { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_275 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_276 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3431,19 +3600,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_10 = dk2_10 * bt_t_10; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; - float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10) + dgf_10; + float qe_9 = q_t0_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dg_o_10 = __fmaf_rn(q_t0_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9)) + dgf_10; dgo_0[2] = dg_o_10; int qoff_10 = t_10 * qrs; if (t_10 < clen) { { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_277 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_278 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3469,19 +3643,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_11 = dk2_11 * bt_t_11; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; - float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11) + dgf_11; + float qe_10 = q_t0_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dg_o_11 = __fmaf_rn(q_t0_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10)) + dgf_11; dgo_0[3] = dg_o_11; int qoff_11 = t_11 * qrs; if (t_11 < clen) { { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_279 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_280 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3507,19 +3686,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_12 = dk2_12 * bt_t_12; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; - float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12) + dgf_12; + float qe_11 = q_t0_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dg_o_12 = __fmaf_rn(q_t0_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11)) + dgf_12; dgo_0[4] = dg_o_12; int qoff_12 = t_12 * qrs; if (t_12 < clen) { { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_281 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_282 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3545,19 +3729,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_13 = dk2_13 * bt_t_13; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; - float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13) + dgf_13; + float qe_12 = q_t0_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dg_o_13 = __fmaf_rn(q_t0_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12)) + dgf_13; dgo_0[5] = dg_o_13; int qoff_13 = t_13 * qrs; if (t_13 < clen) { { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_283 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_284 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3583,19 +3772,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_14 = dk2_14 * bt_t_14; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; - float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14) + dgf_14; + float qe_13 = q_t0_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dg_o_14 = __fmaf_rn(q_t0_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13)) + dgf_14; dgo_0[6] = dg_o_14; int qoff_14 = t_14 * qrs; if (t_14 < clen) { { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_285 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_286 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3621,19 +3815,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_15 = dk2_15 * bt_t_15; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; - float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15) + dgf_15; + float qe_14 = q_t0_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dg_o_15 = __fmaf_rn(q_t0_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14)) + dgf_15; dgo_0[7] = dg_o_15; int qoff_15 = t_15 * qrs; if (t_15 < clen) { { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_287 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_288 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_15; } } float s_run_0[1]; @@ -3662,36 +3861,36 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_289; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_290; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_291; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_292; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_293; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_294; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -3709,9 +3908,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_0 = offset + sfx[0]; if (t_16 < clen) { { - auto _gst_267 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_0; + auto _gst_299 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_0; } } int t_17 = 48 + 8 * th + 1; @@ -3719,9 +3918,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_1 = offset + sfx[1]; if (t_17 < clen) { { - auto _gst_268 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1; + auto _gst_300 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1; } } int t_18 = 48 + 8 * th + 2; @@ -3729,9 +3928,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_2 = offset + sfx[2]; if (t_18 < clen) { { - auto _gst_269 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_2; + auto _gst_301 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_2; } } int t_19 = 48 + 8 * th + 3; @@ -3739,9 +3938,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_3 = offset + sfx[3]; if (t_19 < clen) { { - auto _gst_270 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_3; + auto _gst_302 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_3; } } int t_20 = 48 + 8 * th + 4; @@ -3749,9 +3948,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_4 = offset + sfx[4]; if (t_20 < clen) { { - auto _gst_271 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_4; + auto _gst_303 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_4; } } int t_21 = 48 + 8 * th + 5; @@ -3759,9 +3958,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_5 = offset + sfx[5]; if (t_21 < clen) { { - auto _gst_272 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_5; + auto _gst_304 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_5; } } int t_22 = 48 + 8 * th + 6; @@ -3769,9 +3968,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_6 = offset + sfx[6]; if (t_22 < clen) { { - auto _gst_273 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_6; + auto _gst_305 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_6; } } int t_23 = 48 + 8 * th + 7; @@ -3779,9 +3978,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_7 = offset + sfx[7]; if (t_23 < clen) { { - auto _gst_274 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_7; + auto _gst_306 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3799,9 +3998,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_8 = offset_0 + sfx_0[0]; if (t_24 < clen) { { - auto _gst_275 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_8; + auto _gst_307 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_8; } } int t_25 = 32 + 8 * th + 1; @@ -3809,9 +4008,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_9 = offset_0 + sfx_0[1]; if (t_25 < clen) { { - auto _gst_276 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_9; + auto _gst_308 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_9; } } int t_26 = 32 + 8 * th + 2; @@ -3819,9 +4018,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_10 = offset_0 + sfx_0[2]; if (t_26 < clen) { { - auto _gst_277 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_10; + auto _gst_309 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_10; } } int t_27 = 32 + 8 * th + 3; @@ -3829,9 +4028,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_11 = offset_0 + sfx_0[3]; if (t_27 < clen) { { - auto _gst_278 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_11; + auto _gst_310 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_11; } } int t_28 = 32 + 8 * th + 4; @@ -3839,9 +4038,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_12 = offset_0 + sfx_0[4]; if (t_28 < clen) { { - auto _gst_279 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_12; + auto _gst_311 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_12; } } int t_29 = 32 + 8 * th + 5; @@ -3849,9 +4048,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_13 = offset_0 + sfx_0[5]; if (t_29 < clen) { { - auto _gst_280 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_13; + auto _gst_312 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_13; } } int t_30 = 32 + 8 * th + 6; @@ -3859,9 +4058,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_14 = offset_0 + sfx_0[6]; if (t_30 < clen) { { - auto _gst_281 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_14; + auto _gst_313 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_14; } } int t_31 = 32 + 8 * th + 7; @@ -3869,15 +4068,15 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_15 = offset_0 + sfx_0[7]; if (t_31 < clen) { { - auto _gst_282 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_15; + auto _gst_314 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3936,19 +4135,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_16 = dk2_16 * bt_t_16; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; - float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16) + dgf_16; + float qe_15 = q_t0_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dg_o_16 = __fmaf_rn(q_t0_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15)) + dgf_16; dgo_1[0] = dg_o_16; int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_284 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_316 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_285 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_317 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_33 = 16 + 8 * th + 1; @@ -3974,19 +4178,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_17 = dk2_17 * bt_t_17; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; - float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17) + dgf_17; + float qe_16 = q_t0_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dg_o_17 = __fmaf_rn(q_t0_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16)) + dgf_17; dgo_1[1] = dg_o_17; int qoff_17 = t_33 * qrs; if (t_33 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_34 = 16 + 8 * th + 2; @@ -4012,19 +4221,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_18 = dk2_18 * bt_t_18; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; - float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18) + dgf_18; + float qe_17 = q_t0_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dg_o_18 = __fmaf_rn(q_t0_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17)) + dgf_18; dgo_1[2] = dg_o_18; int qoff_18 = t_34 * qrs; if (t_34 < clen) { { - auto _gst_288 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_320 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_289 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_321 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_35 = 16 + 8 * th + 3; @@ -4050,19 +4264,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_19 = dk2_19 * bt_t_19; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; - float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19) + dgf_19; + float qe_18 = q_t0_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dg_o_19 = __fmaf_rn(q_t0_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18)) + dgf_19; dgo_1[3] = dg_o_19; int qoff_19 = t_35 * qrs; if (t_35 < clen) { { - auto _gst_290 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_322 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_291 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_323 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_36 = 16 + 8 * th + 4; @@ -4088,19 +4307,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_20 = dk2_20 * bt_t_20; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; - float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20) + dgf_20; + float qe_19 = q_t0_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dg_o_20 = __fmaf_rn(q_t0_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19)) + dgf_20; dgo_1[4] = dg_o_20; int qoff_20 = t_36 * qrs; if (t_36 < clen) { { - auto _gst_292 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_324 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_293 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_325 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_37 = 16 + 8 * th + 5; @@ -4126,19 +4350,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_21 = dk2_21 * bt_t_21; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; - float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21) + dgf_21; + float qe_20 = q_t0_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dg_o_21 = __fmaf_rn(q_t0_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20)) + dgf_21; dgo_1[5] = dg_o_21; int qoff_21 = t_37 * qrs; if (t_37 < clen) { { - auto _gst_294 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_326 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_295 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_327 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_38 = 16 + 8 * th + 6; @@ -4164,19 +4393,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_22 = dk2_22 * bt_t_22; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; - float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22) + dgf_22; + float qe_21 = q_t0_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dg_o_22 = __fmaf_rn(q_t0_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21)) + dgf_22; dgo_1[6] = dg_o_22; int qoff_22 = t_38 * qrs; if (t_38 < clen) { { - auto _gst_296 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_328 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_297 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_329 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_39 = 16 + 8 * th + 7; @@ -4202,19 +4436,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_23 = dk2_23 * bt_t_23; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; - float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23) + dgf_23; + float qe_22 = q_t0_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dg_o_23 = __fmaf_rn(q_t0_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22)) + dgf_23; dgo_1[7] = dg_o_23; int qoff_23 = t_39 * qrs; if (t_39 < clen) { { - auto _gst_298 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_330 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_299 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_331 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_23; } } float s_run_1[1]; @@ -4243,37 +4482,37 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_332; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_333; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_334; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_335; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_336; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_337; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4291,9 +4530,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_16 = offset_1 + sfx_1[0]; if (t_40 < clen) { { - auto _gst_310 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_16; + auto _gst_342 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_16; } } int t_41 = 16 + 8 * th + 1; @@ -4301,9 +4540,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_17 = offset_1 + sfx_1[1]; if (t_41 < clen) { { - auto _gst_311 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_17; + auto _gst_343 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_17; } } int t_42 = 16 + 8 * th + 2; @@ -4311,9 +4550,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_18 = offset_1 + sfx_1[2]; if (t_42 < clen) { { - auto _gst_312 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_18; + auto _gst_344 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_18; } } int t_43 = 16 + 8 * th + 3; @@ -4321,9 +4560,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_19 = offset_1 + sfx_1[3]; if (t_43 < clen) { { - auto _gst_313 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_19; + auto _gst_345 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_19; } } int t_44 = 16 + 8 * th + 4; @@ -4331,9 +4570,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_20 = offset_1 + sfx_1[4]; if (t_44 < clen) { { - auto _gst_314 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_20; + auto _gst_346 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_20; } } int t_45 = 16 + 8 * th + 5; @@ -4341,9 +4580,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_21 = offset_1 + sfx_1[5]; if (t_45 < clen) { { - auto _gst_315 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_21; + auto _gst_347 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_21; } } int t_46 = 16 + 8 * th + 6; @@ -4351,9 +4590,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_22 = offset_1 + sfx_1[6]; if (t_46 < clen) { { - auto _gst_316 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_22; + auto _gst_348 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_22; } } int t_47 = 16 + 8 * th + 7; @@ -4361,15 +4600,15 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_23 = offset_1 + sfx_1[7]; if (t_47 < clen) { { - auto _gst_317 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_23; + auto _gst_349 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4422,19 +4661,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_24 = dk2_24 * bt_t_24; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; - float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24) + dgf_24; + float qe_23 = q_t0_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dg_o_24 = __fmaf_rn(q_t0_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23)) + dgf_24; dgo_2[0] = dg_o_24; int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_319 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_351 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_320 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_352 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_49 = 8 * th + 1; @@ -4460,19 +4704,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_25 = dk2_25 * bt_t_25; float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; - float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25) + dgf_25; + float qe_24 = q_t0_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dg_o_25 = __fmaf_rn(q_t0_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24)) + dgf_25; dgo_2[1] = dg_o_25; int qoff_25 = t_49 * qrs; if (t_49 < clen) { { - auto _gst_321 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_353 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_322 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_354 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_50 = 8 * th + 2; @@ -4498,19 +4747,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_26 = dk2_26 * bt_t_26; float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; - float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26) + dgf_26; + float qe_25 = q_t0_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dg_o_26 = __fmaf_rn(q_t0_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25)) + dgf_26; dgo_2[2] = dg_o_26; int qoff_26 = t_50 * qrs; if (t_50 < clen) { { - auto _gst_323 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_355 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_324 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_356 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_51 = 8 * th + 3; @@ -4536,19 +4790,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_27 = dk2_27 * bt_t_27; float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; - float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27) + dgf_27; + float qe_26 = q_t0_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dg_o_27 = __fmaf_rn(q_t0_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26)) + dgf_27; dgo_2[3] = dg_o_27; int qoff_27 = t_51 * qrs; if (t_51 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_52 = 8 * th + 4; @@ -4574,19 +4833,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_28 = dk2_28 * bt_t_28; float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; - float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28) + dgf_28; + float qe_27 = q_t0_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dg_o_28 = __fmaf_rn(q_t0_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27)) + dgf_28; dgo_2[4] = dg_o_28; int qoff_28 = t_52 * qrs; if (t_52 < clen) { { - auto _gst_327 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_359 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_328 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_360 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_53 = 8 * th + 5; @@ -4612,19 +4876,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_29 = dk2_29 * bt_t_29; float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; - float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29) + dgf_29; + float qe_28 = q_t0_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dg_o_29 = __fmaf_rn(q_t0_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28)) + dgf_29; dgo_2[5] = dg_o_29; int qoff_29 = t_53 * qrs; if (t_53 < clen) { { - auto _gst_329 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_361 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_330 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_362 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_54 = 8 * th + 6; @@ -4650,19 +4919,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_30 = dk2_30 * bt_t_30; float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; - float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30) + dgf_30; + float qe_29 = q_t0_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dg_o_30 = __fmaf_rn(q_t0_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29)) + dgf_30; dgo_2[6] = dg_o_30; int qoff_30 = t_54 * qrs; if (t_54 < clen) { { - auto _gst_331 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_363 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_332 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_364 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_55 = 8 * th + 7; @@ -4688,19 +4962,24 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dk2b_31 = dk2_31 * bt_t_31; float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; - float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31) + dgf_31; + float qe_30 = q_t0_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dg_o_31 = __fmaf_rn(q_t0_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30)) + dgf_31; dgo_2[7] = dg_o_31; int qoff_31 = t_55 * qrs; if (t_55 < clen) { { - auto _gst_333 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_365 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_334 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_366 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_31; } } float s_run_2[1]; @@ -4729,35 +5008,35 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_367; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_368; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_369; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_370; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_371; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_372; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_2) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4775,9 +5054,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_24 = offset_2 + sfx_2[0]; if (t_56 < clen) { { - auto _gst_345 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_24; + auto _gst_377 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_24; } } int t_57 = 8 * th + 1; @@ -4785,9 +5064,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_25 = offset_2 + sfx_2[1]; if (t_57 < clen) { { - auto _gst_346 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_25; + auto _gst_378 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_25; } } int t_58 = 8 * th + 2; @@ -4795,9 +5074,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_26 = offset_2 + sfx_2[2]; if (t_58 < clen) { { - auto _gst_347 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_26; + auto _gst_379 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_26; } } int t_59 = 8 * th + 3; @@ -4805,9 +5084,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_27 = offset_2 + sfx_2[3]; if (t_59 < clen) { { - auto _gst_348 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_27; + auto _gst_380 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_27; } } int t_60 = 8 * th + 4; @@ -4815,9 +5094,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_28 = offset_2 + sfx_2[4]; if (t_60 < clen) { { - auto _gst_349 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_28; + auto _gst_381 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_28; } } int t_61 = 8 * th + 5; @@ -4825,9 +5104,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_29 = offset_2 + sfx_2[5]; if (t_61 < clen) { { - auto _gst_350 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_29; + auto _gst_382 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_29; } } int t_62 = 8 * th + 6; @@ -4835,9 +5114,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_30 = offset_2 + sfx_2[6]; if (t_62 < clen) { { - auto _gst_351 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_30; + auto _gst_383 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_30; } } int t_63 = 8 * th + 7; @@ -4845,9 +5124,9 @@ kernel_cake_kda_train_c27b09eb5f5881be2969(const __grid_constant__ CUtensorMap d float dg_31 = offset_2 + sfx_2[7]; if (t_63 < clen) { { - auto _gst_352 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_31; + auto _gst_384 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_31; } } off_acc[0] = off_acc[0] + t_hi_2 + t_lo_2; diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu index 637fa7a75e0..9fa69d65b88 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_ea14e16d1fc71364ba03_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -199,6 +200,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2267,12 +2284,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_57); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_0); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_0 : __fmaf_rn(dl, _rcp_170, e2_0)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2296,12 +2319,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_58); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_1); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_1, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_1, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_1 : __fmaf_rn(dl_1, _rcp_174, e2_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2325,12 +2354,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_59); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_2); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_2, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_2, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_2 : __fmaf_rn(dl_3, _rcp_178, e2_2)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2354,12 +2389,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_60); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_3); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_3, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_3, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_3 : __fmaf_rn(dl_5, _rcp_182, e2_3)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2395,12 +2436,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2420,12 +2467,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2445,12 +2498,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2470,12 +2529,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2523,12 +2588,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_0 = sub_f32x2(_f2_64, _f2_65); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2552,12 +2623,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_1 = sub_f32x2(_f2_64, _f2_66); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2581,12 +2658,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_2 = sub_f32x2(_f2_64, _f2_67); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2610,12 +2693,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_3 = sub_f32x2(_f2_64, _f2_68); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2657,12 +2746,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_4 = sub_f32x2(_f2_70, _f2_71); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2686,12 +2781,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_5 = sub_f32x2(_f2_70, _f2_72); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2715,12 +2816,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_6 = sub_f32x2(_f2_70, _f2_73); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2744,12 +2851,18 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 dn2_7 = sub_f32x2(_f2_70, _f2_74); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2820,37 +2933,37 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float2 d2_56 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_56.x); float e1_12 = approx_exp2(d2_56.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_57 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_57.x); float e1_13 = approx_exp2(d2_57.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_58 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_58.x); float e1_14 = approx_exp2(d2_58.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_59 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_59.x); float e1_15 = approx_exp2(d2_59.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2862,29 +2975,29 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2892,29 +3005,29 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2933,409 +3046,145 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float r_247[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_247[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 4096 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 4096 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); float gn = r_247[0]; - float gc_888 = gn - 21.640425f; + float gc_0 = gn - 21.640425f; int eb = 98304 + th * -32768; int ew = eb >> 2; - int qbase_889 = pool_f32_addr + (unsigned int)(eb + 12288 + dh_lane * 1024); + int qbase_0 = pool_f32_addr + (unsigned int)(eb + 12288 + dh_lane * 1024); int sb8 = 32 + 8 * th; int gb8 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8 * 128); int kb8 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8 * 128); - float dbp[8]; - float _tmem_load_7[8]; - tmem_ld_x8(&_tmem_load_7[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float _tmem_load_8[8]; - tmem_ld_x8(&_tmem_load_8[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float _tmem_load_9[8]; - tmem_ld_x8(&_tmem_load_9[0], taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)); - float _tmem_load_10[8]; - tmem_ld_x8(&_tmem_load_10[0], taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th)); - float _tmem_load_11[8]; - tmem_ld_x8(&_tmem_load_11[0], taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th)); - float btv[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[0])), "=r"(*reinterpret_cast(&btv[(0) + 1])), "=r"(*reinterpret_cast(&btv[(0) + 2])), "=r"(*reinterpret_cast(&btv[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[4])), "=r"(*reinterpret_cast(&btv[(4) + 1])), "=r"(*reinterpret_cast(&btv[(4) + 2])), "=r"(*reinterpret_cast(&btv[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int c_qa = 128 + 8 * th; + int c_ka = 144 + 8 * th; + int c_qd = 192 + 8 * th; + int c_kd = 208 + 8 * th; + int c_dk = 32 + 8 * th; + int c_bp = c_ka; float s_run[1]; s_run[0] = 0.0f; - int t_1_0 = 32 + 8 * th + 7; - float r_248[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_0 = r_248[0]; - unsigned int r_249[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_0 = __uint_as_float(r_249[0] << 16); - unsigned int r_250[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_0 = __uint_as_float(r_250[0] << 16); - float q_t_2_0 = ((t_1_0 >= clen) ? 0.0f : q_t_1_0); - float dqf_1_0 = pool_f32[ew + 896 + d_lane]; - float dkf_1_0 = pool_f32[ew + 1920 + d_lane]; - float dgf_1_0 = pool_f32[ew + 2944 + d_lane]; - float bt_t_1_0 = btv[7]; - float e_off_1_0 = approx_exp2(g_t_1_0 - gn); - float e_dg_1_0 = approx_exp2(g_t_1_0 - gc_888); - float dq2_1_0 = __fmaf_rn(_tmem_load_7[7], e_off_1_0, _tmem_load_9[7] * e_dg_1_0); - float dk2_1_0 = __fmaf_rn(_tmem_load_8[7], e_off_1_0, _tmem_load_10[7] * e_dg_1_0); - dbp[7] = dk2_1_0 * k_t_1_0; - float dk2b_1_0 = dk2_1_0 * bt_t_1_0; - float dq_o_1_0 = dq2_1_0 + dqf_1_0; - float dk_o_1_0 = dk2b_1_0 + dkf_1_0 + _tmem_load_11[7]; - float dg_o_1_0 = __fmaf_rn(q_t_2_0, dq2_1_0, (dk2b_1_0 - _tmem_load_11[7]) * k_t_1_0) + dgf_1_0; - s_run[0] = s_run[0] + dg_o_1_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 7)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_0 = t_1_0 * qrs; - if (t_1_0 < clen) { - { - auto _gst_216 = dq_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_1_0; - } - { - auto _gst_217 = dk_base + qoff_1_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_1_0; - } - } - int t_1_1 = 32 + 8 * th + 6; - float r_251[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_1 = r_251[0]; - unsigned int r_252[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((kb8 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_1 = __uint_as_float(r_252[0] << 16); - unsigned int r_253[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_1 = __uint_as_float(r_253[0] << 16); - float q_t_2_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_1); - float dqf_1_1 = pool_f32[ew + 768 + d_lane]; - float dkf_1_1 = pool_f32[ew + 1792 + d_lane]; - float dgf_1_1 = pool_f32[ew + 2816 + d_lane]; - float bt_t_1_1 = btv[6]; - float e_off_1_1 = approx_exp2(g_t_1_1 - gn); - float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_888); - float dq2_1_1 = __fmaf_rn(_tmem_load_7[6], e_off_1_1, _tmem_load_9[6] * e_dg_1_1); - float dk2_1_1 = __fmaf_rn(_tmem_load_8[6], e_off_1_1, _tmem_load_10[6] * e_dg_1_1); - dbp[6] = dk2_1_1 * k_t_1_1; - float dk2b_1_1 = dk2_1_1 * bt_t_1_1; - float dq_o_1_1 = dq2_1_1 + dqf_1_1; - float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_11[6]; - float dg_o_1_1 = __fmaf_rn(q_t_2_1, dq2_1_1, (dk2b_1_1 - _tmem_load_11[6]) * k_t_1_1) + dgf_1_1; - s_run[0] = s_run[0] + dg_o_1_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 6)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_1 = t_1_1 * qrs; - if (t_1_1 < clen) { - { - auto _gst_218 = dq_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1_1; - } - { - auto _gst_219 = dk_base + qoff_1_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1_1; - } - } - int t_1_2 = 32 + 8 * th + 5; - float r_254[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_2 = r_254[0]; - unsigned int r_255[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((kb8 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_2 = __uint_as_float(r_255[0] << 16); - unsigned int r_256[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_2 = __uint_as_float(r_256[0] << 16); - float q_t_2_2 = ((t_1_2 >= clen) ? 0.0f : q_t_1_2); - float dqf_1_2 = pool_f32[ew + 640 + d_lane]; - float dkf_1_2 = pool_f32[ew + 1664 + d_lane]; - float dgf_1_2 = pool_f32[ew + 2688 + d_lane]; - float bt_t_1_2 = btv[5]; - float e_off_1_2 = approx_exp2(g_t_1_2 - gn); - float e_dg_1_2 = approx_exp2(g_t_1_2 - gc_888); - float dq2_1_2 = __fmaf_rn(_tmem_load_7[5], e_off_1_2, _tmem_load_9[5] * e_dg_1_2); - float dk2_1_2 = __fmaf_rn(_tmem_load_8[5], e_off_1_2, _tmem_load_10[5] * e_dg_1_2); - dbp[5] = dk2_1_2 * k_t_1_2; - float dk2b_1_2 = dk2_1_2 * bt_t_1_2; - float dq_o_1_2 = dq2_1_2 + dqf_1_2; - float dk_o_1_2 = dk2b_1_2 + dkf_1_2 + _tmem_load_11[5]; - float dg_o_1_2 = __fmaf_rn(q_t_2_2, dq2_1_2, (dk2b_1_2 - _tmem_load_11[5]) * k_t_1_2) + dgf_1_2; - s_run[0] = s_run[0] + dg_o_1_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 5)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_2 = t_1_2 * qrs; - if (t_1_2 < clen) { - { - auto _gst_220 = dq_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1_2; - } - { - auto _gst_221 = dk_base + qoff_1_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1_2; - } - } - int t_1_3 = 32 + 8 * th + 4; - float r_257[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_3 = r_257[0]; - unsigned int r_258[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((kb8 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_3 = __uint_as_float(r_258[0] << 16); - unsigned int r_259[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_3 = __uint_as_float(r_259[0] << 16); - float q_t_2_3 = ((t_1_3 >= clen) ? 0.0f : q_t_1_3); - float dqf_1_3 = pool_f32[ew + 512 + d_lane]; - float dkf_1_3 = pool_f32[ew + 1536 + d_lane]; - float dgf_1_3 = pool_f32[ew + 2560 + d_lane]; - float bt_t_1_3 = btv[4]; - float e_off_1_3 = approx_exp2(g_t_1_3 - gn); - float e_dg_1_3 = approx_exp2(g_t_1_3 - gc_888); - float dq2_1_3 = __fmaf_rn(_tmem_load_7[4], e_off_1_3, _tmem_load_9[4] * e_dg_1_3); - float dk2_1_3 = __fmaf_rn(_tmem_load_8[4], e_off_1_3, _tmem_load_10[4] * e_dg_1_3); - dbp[4] = dk2_1_3 * k_t_1_3; - float dk2b_1_3 = dk2_1_3 * bt_t_1_3; - float dq_o_1_3 = dq2_1_3 + dqf_1_3; - float dk_o_1_3 = dk2b_1_3 + dkf_1_3 + _tmem_load_11[4]; - float dg_o_1_3 = __fmaf_rn(q_t_2_3, dq2_1_3, (dk2b_1_3 - _tmem_load_11[4]) * k_t_1_3) + dgf_1_3; - s_run[0] = s_run[0] + dg_o_1_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 4)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_3 = t_1_3 * qrs; - if (t_1_3 < clen) { - { - auto _gst_222 = dq_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_1_3; - } - { - auto _gst_223 = dk_base + qoff_1_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_1_3; - } - } - int t_1_4 = 32 + 8 * th + 3; - float r_260[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_4 = r_260[0]; - unsigned int r_261[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((kb8 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_4 = __uint_as_float(r_261[0] << 16); - unsigned int r_262[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_4 = __uint_as_float(r_262[0] << 16); - float q_t_2_4 = ((t_1_4 >= clen) ? 0.0f : q_t_1_4); - float dqf_1_4 = pool_f32[ew + 384 + d_lane]; - float dkf_1_4 = pool_f32[ew + 1408 + d_lane]; - float dgf_1_4 = pool_f32[ew + 2432 + d_lane]; - float bt_t_1_4 = btv[3]; - float e_off_1_4 = approx_exp2(g_t_1_4 - gn); - float e_dg_1_4 = approx_exp2(g_t_1_4 - gc_888); - float dq2_1_4 = __fmaf_rn(_tmem_load_7[3], e_off_1_4, _tmem_load_9[3] * e_dg_1_4); - float dk2_1_4 = __fmaf_rn(_tmem_load_8[3], e_off_1_4, _tmem_load_10[3] * e_dg_1_4); - dbp[3] = dk2_1_4 * k_t_1_4; - float dk2b_1_4 = dk2_1_4 * bt_t_1_4; - float dq_o_1_4 = dq2_1_4 + dqf_1_4; - float dk_o_1_4 = dk2b_1_4 + dkf_1_4 + _tmem_load_11[3]; - float dg_o_1_4 = __fmaf_rn(q_t_2_4, dq2_1_4, (dk2b_1_4 - _tmem_load_11[3]) * k_t_1_4) + dgf_1_4; - s_run[0] = s_run[0] + dg_o_1_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 3)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_4 = t_1_4 * qrs; - if (t_1_4 < clen) { - { - auto _gst_224 = dq_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_1_4; - } - { - auto _gst_225 = dk_base + qoff_1_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_1_4; - } - } - int t_1_5 = 32 + 8 * th + 2; - float r_263[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_5 = r_263[0]; - unsigned int r_264[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((kb8 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_5 = __uint_as_float(r_264[0] << 16); - unsigned int r_265[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_5 = __uint_as_float(r_265[0] << 16); - float q_t_2_5 = ((t_1_5 >= clen) ? 0.0f : q_t_1_5); - float dqf_1_5 = pool_f32[ew + 256 + d_lane]; - float dkf_1_5 = pool_f32[ew + 1280 + d_lane]; - float dgf_1_5 = pool_f32[ew + 2304 + d_lane]; - float bt_t_1_5 = btv[2]; - float e_off_1_5 = approx_exp2(g_t_1_5 - gn); - float e_dg_1_5 = approx_exp2(g_t_1_5 - gc_888); - float dq2_1_5 = __fmaf_rn(_tmem_load_7[2], e_off_1_5, _tmem_load_9[2] * e_dg_1_5); - float dk2_1_5 = __fmaf_rn(_tmem_load_8[2], e_off_1_5, _tmem_load_10[2] * e_dg_1_5); - dbp[2] = dk2_1_5 * k_t_1_5; - float dk2b_1_5 = dk2_1_5 * bt_t_1_5; - float dq_o_1_5 = dq2_1_5 + dqf_1_5; - float dk_o_1_5 = dk2b_1_5 + dkf_1_5 + _tmem_load_11[2]; - float dg_o_1_5 = __fmaf_rn(q_t_2_5, dq2_1_5, (dk2b_1_5 - _tmem_load_11[2]) * k_t_1_5) + dgf_1_5; - s_run[0] = s_run[0] + dg_o_1_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 2)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_5 = t_1_5 * qrs; - if (t_1_5 < clen) { - { - auto _gst_226 = dq_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_1_5; - } - { - auto _gst_227 = dk_base + qoff_1_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_1_5; - } - } - int t_1_6 = 32 + 8 * th + 1; - float r_266[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_6 = r_266[0]; - unsigned int r_267[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_6 = __uint_as_float(r_267[0] << 16); - unsigned int r_268[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_6 = __uint_as_float(r_268[0] << 16); - float q_t_2_6 = ((t_1_6 >= clen) ? 0.0f : q_t_1_6); - float dqf_1_6 = pool_f32[ew + 128 + d_lane]; - float dkf_1_6 = pool_f32[ew + 1152 + d_lane]; - float dgf_1_6 = pool_f32[ew + 2176 + d_lane]; - float bt_t_1_6 = btv[1]; - float e_off_1_6 = approx_exp2(g_t_1_6 - gn); - float e_dg_1_6 = approx_exp2(g_t_1_6 - gc_888); - float dq2_1_6 = __fmaf_rn(_tmem_load_7[1], e_off_1_6, _tmem_load_9[1] * e_dg_1_6); - float dk2_1_6 = __fmaf_rn(_tmem_load_8[1], e_off_1_6, _tmem_load_10[1] * e_dg_1_6); - dbp[1] = dk2_1_6 * k_t_1_6; - float dk2b_1_6 = dk2_1_6 * bt_t_1_6; - float dq_o_1_6 = dq2_1_6 + dqf_1_6; - float dk_o_1_6 = dk2b_1_6 + dkf_1_6 + _tmem_load_11[1]; - float dg_o_1_6 = __fmaf_rn(q_t_2_6, dq2_1_6, (dk2b_1_6 - _tmem_load_11[1]) * k_t_1_6) + dgf_1_6; - s_run[0] = s_run[0] + dg_o_1_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 1)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_6 = t_1_6 * qrs; - if (t_1_6 < clen) { - { - auto _gst_228 = dq_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_1_6; - } - { - auto _gst_229 = dk_base + qoff_1_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_1_6; - } - } - int t_1_7 = 32 + 8 * th; - float r_269[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1_7 = r_269[0]; - unsigned int r_270[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((kb8 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1_7 = __uint_as_float(r_270[0] << 16); - unsigned int r_271[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((qbase_889 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1_7 = __uint_as_float(r_271[0] << 16); - float q_t_2_7 = ((t_1_7 >= clen) ? 0.0f : q_t_1_7); - float dqf_1_7 = pool_f32[ew + d_lane]; - float dkf_1_7 = pool_f32[ew + 1024 + d_lane]; - float dgf_1_7 = pool_f32[ew + 2048 + d_lane]; - float bt_t_1_7 = btv[0]; - float e_off_1_7 = approx_exp2(g_t_1_7 - gn); - float e_dg_1_7 = approx_exp2(g_t_1_7 - gc_888); - float dq2_1_7 = __fmaf_rn(_tmem_load_7[0], e_off_1_7, _tmem_load_9[0] * e_dg_1_7); - float dk2_1_7 = __fmaf_rn(_tmem_load_8[0], e_off_1_7, _tmem_load_10[0] * e_dg_1_7); - dbp[0] = dk2_1_7 * k_t_1_7; - float dk2b_1_7 = dk2_1_7 * bt_t_1_7; - float dq_o_1_7 = dq2_1_7 + dqf_1_7; - float dk_o_1_7 = dk2b_1_7 + dkf_1_7 + _tmem_load_11[0]; - float dg_o_1_7 = __fmaf_rn(q_t_2_7, dq2_1_7, (dk2b_1_7 - _tmem_load_11[0]) * k_t_1_7) + dgf_1_7; - s_run[0] = s_run[0] + dg_o_1_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1_7 = t_1_7 * qrs; - if (t_1_7 < clen) { - { - auto _gst_230 = dq_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_1_7; - } - { - auto _gst_231 = dk_base + qoff_1_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_1_7; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j = 7 - k; + float a_qd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j))); + float a_kd[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd + j))); + float a_dk[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk + j))); + float a_qa[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa + j))); + float a_ka[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka + j))); + float btr[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8 + j)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8 + j; + float r_248[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_248[0]; + unsigned int r_249[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_249[0] << 16); + unsigned int r_250[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_250[0] << 16); + float q_m = ((t >= clen) ? 0.0f : q_t); + float dqf = pool_f32[ew + 128 * j + d_lane]; + float dkf = pool_f32[ew + (1024 + 128 * j) + d_lane]; + float dgf = pool_f32[ew + (2048 + 128 * j) + d_lane]; + float bt_t = btr[0]; + float e_dg = approx_exp2(g_t - gc_0); + float e_off = approx_exp2(g_t - gn); + float dq2 = __fmaf_rn(a_qa[0], e_off, a_qd[0] * e_dg); + float dk2 = __fmaf_rn(a_ka[0], e_off, a_kd[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + a_dk[0]; + float qe = q_m * e_dg; + float kp = k_t * bt_t; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, a_qd[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg) * a_kd[0]); + float dg_o = __fmaf_rn(q_m, dq2, __fmaf_rn(dk2b - a_dk[0], k_t, cr)) + dgf; + s_run[0] = s_run[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd + j)), "f"(s_run[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff = t * qrs; + if (t < clen) { + { + auto _gst_248 = dq_base + qoff; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o; + } + { + auto _gst_249 = dk_base + qoff; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o; + } } } float tot = s_run[0]; + float dbp[8]; + tmem_ld_x8(&dbp[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); int b4 = lane_1 >> 4 & 1; int b3 = lane_1 >> 3 & 1; int b2 = lane_1 >> 2 & 1; float v4[4]; - float send_1_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_1_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; - float send_1_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; - float send_1_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_1_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; - float send_1_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_1_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; - float v2_1044[2]; - float send_1_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_1_4, 8); - v2_1044[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; - float send_1_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_1_5, 8); - v2_1044[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; - float send_1047 = ((b2 == 0) ? v2_1044[1] : v2_1044[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send_1047, 4); - float v1 = ((b2 == 0) ? v2_1044[0] : v2_1044[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_250; + float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_251; + float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_252; + float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_253; + float v2[2]; + float send2_0 = ((b3 == 0) ? v4[2] : v4[0]); + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send2_0, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_254; + float send2_1 = ((b3 == 0) ? v4[3] : v4[1]); + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send2_1, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_255; + float send1 = ((b2 == 0) ? v2[1] : v2[0]); + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send1, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_256; + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_257; + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_258; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3346,412 +3195,148 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d if (lane_1 == 17) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(dt_bias + (hv * 128 + 32 * qd)))); } mbarrier_wait(epi_full_addr + (th) * 8, 1); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_272[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1049 = r_272[0]; - float gc_1050 = gn_1049 - 21.640425f; - int eb_1051 = 98304 + th * -32768; - int ew_1052 = eb_1051 >> 2; - int qbase_1053 = pool_f32_addr + (unsigned int)(eb_1051 + 12288 + dh_lane * 1024); - int sb8_1054 = 48 + 8 * th; - int gb8_1055 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1054 * 128); - int kb8_1056 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1054 * 128); - float dbp_1057[8]; - float _tmem_load_12[8]; - tmem_ld_x8(&_tmem_load_12[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float _tmem_load_13[8]; - tmem_ld_x8(&_tmem_load_13[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float _tmem_load_14[8]; - tmem_ld_x8(&_tmem_load_14[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float _tmem_load_15[8]; - tmem_ld_x8(&_tmem_load_15[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float _tmem_load_16[8]; - tmem_ld_x8(&_tmem_load_16[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_1058[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[0])), "=r"(*reinterpret_cast(&btv_1058[(0) + 1])), "=r"(*reinterpret_cast(&btv_1058[(0) + 2])), "=r"(*reinterpret_cast(&btv_1058[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1058[4])), "=r"(*reinterpret_cast(&btv_1058[(4) + 1])), "=r"(*reinterpret_cast(&btv_1058[(4) + 2])), "=r"(*reinterpret_cast(&btv_1058[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_1059[1]; - s_run_1059[0] = 0.0f; - int t_0 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew_1052 + 896 + d_lane]; - float dkf_0 = pool_f32[ew_1052 + 1920 + d_lane]; - float dgf_0 = pool_f32[ew_1052 + 2944 + d_lane]; - float bt_t_0 = btv_1058[7]; - float e_off_0 = approx_exp2(g_t_0 - gn_1049); - float e_dg_0 = approx_exp2(g_t_0 - gc_1050); - float dq2_0 = __fmaf_rn(_tmem_load_12[7], e_off_0, _tmem_load_14[7] * e_dg_0); - float dk2_0 = __fmaf_rn(_tmem_load_13[7], e_off_0, _tmem_load_15[7] * e_dg_0); - dbp_1057[7] = dk2_0 * k_t_0; - float dk2b_0 = dk2_0 * bt_t_0; - float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_16[7]; - float dg_o_0 = __fmaf_rn(q_t_1_8, dq2_0, (dk2b_0 - _tmem_load_16[7]) * k_t_0) + dgf_0; - s_run_1059[0] = s_run_1059[0] + dg_o_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_0 = t_0 * qrs; - if (t_0 < clen) { - { - auto _gst_241 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_0; - } - { - auto _gst_242 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_0; - } - } - int t_1 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew_1052 + 768 + d_lane]; - float dkf_1 = pool_f32[ew_1052 + 1792 + d_lane]; - float dgf_1 = pool_f32[ew_1052 + 2816 + d_lane]; - float bt_t_1 = btv_1058[6]; - float e_off_1 = approx_exp2(g_t_1 - gn_1049); - float e_dg_1 = approx_exp2(g_t_1 - gc_1050); - float dq2_1 = __fmaf_rn(_tmem_load_12[6], e_off_1, _tmem_load_14[6] * e_dg_1); - float dk2_1 = __fmaf_rn(_tmem_load_13[6], e_off_1, _tmem_load_15[6] * e_dg_1); - dbp_1057[6] = dk2_1 * k_t_1; - float dk2b_1 = dk2_1 * bt_t_1; - float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + _tmem_load_16[6]; - float dg_o_1 = __fmaf_rn(q_t_1_9, dq2_1, (dk2b_1 - _tmem_load_16[6]) * k_t_1) + dgf_1; - s_run_1059[0] = s_run_1059[0] + dg_o_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1 = t_1 * qrs; - if (t_1 < clen) { - { - auto _gst_243 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_1; - } - { - auto _gst_244 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_1; - } - } - int t_2 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew_1052 + 640 + d_lane]; - float dkf_2 = pool_f32[ew_1052 + 1664 + d_lane]; - float dgf_2 = pool_f32[ew_1052 + 2688 + d_lane]; - float bt_t_2 = btv_1058[5]; - float e_off_2 = approx_exp2(g_t_2 - gn_1049); - float e_dg_2 = approx_exp2(g_t_2 - gc_1050); - float dq2_2 = __fmaf_rn(_tmem_load_12[5], e_off_2, _tmem_load_14[5] * e_dg_2); - float dk2_2 = __fmaf_rn(_tmem_load_13[5], e_off_2, _tmem_load_15[5] * e_dg_2); - dbp_1057[5] = dk2_2 * k_t_2; - float dk2b_2 = dk2_2 * bt_t_2; - float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_16[5]; - float dg_o_2 = __fmaf_rn(q_t_1_10, dq2_2, (dk2b_2 - _tmem_load_16[5]) * k_t_2) + dgf_2; - s_run_1059[0] = s_run_1059[0] + dg_o_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_2 = t_2 * qrs; - if (t_2 < clen) { - { - auto _gst_245 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_2; - } - { - auto _gst_246 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_2; - } - } - int t_3 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew_1052 + 512 + d_lane]; - float dkf_3 = pool_f32[ew_1052 + 1536 + d_lane]; - float dgf_3 = pool_f32[ew_1052 + 2560 + d_lane]; - float bt_t_3 = btv_1058[4]; - float e_off_3 = approx_exp2(g_t_3 - gn_1049); - float e_dg_3 = approx_exp2(g_t_3 - gc_1050); - float dq2_3 = __fmaf_rn(_tmem_load_12[4], e_off_3, _tmem_load_14[4] * e_dg_3); - float dk2_3 = __fmaf_rn(_tmem_load_13[4], e_off_3, _tmem_load_15[4] * e_dg_3); - dbp_1057[4] = dk2_3 * k_t_3; - float dk2b_3 = dk2_3 * bt_t_3; - float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_16[4]; - float dg_o_3 = __fmaf_rn(q_t_1_11, dq2_3, (dk2b_3 - _tmem_load_16[4]) * k_t_3) + dgf_3; - s_run_1059[0] = s_run_1059[0] + dg_o_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_3 = t_3 * qrs; - if (t_3 < clen) { - { - auto _gst_247 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_3; - } - { - auto _gst_248 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_3; - } - } - int t_4 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew_1052 + 384 + d_lane]; - float dkf_4 = pool_f32[ew_1052 + 1408 + d_lane]; - float dgf_4 = pool_f32[ew_1052 + 2432 + d_lane]; - float bt_t_4 = btv_1058[3]; - float e_off_4 = approx_exp2(g_t_4 - gn_1049); - float e_dg_4 = approx_exp2(g_t_4 - gc_1050); - float dq2_4 = __fmaf_rn(_tmem_load_12[3], e_off_4, _tmem_load_14[3] * e_dg_4); - float dk2_4 = __fmaf_rn(_tmem_load_13[3], e_off_4, _tmem_load_15[3] * e_dg_4); - dbp_1057[3] = dk2_4 * k_t_4; - float dk2b_4 = dk2_4 * bt_t_4; - float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_16[3]; - float dg_o_4 = __fmaf_rn(q_t_1_12, dq2_4, (dk2b_4 - _tmem_load_16[3]) * k_t_4) + dgf_4; - s_run_1059[0] = s_run_1059[0] + dg_o_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_4 = t_4 * qrs; - if (t_4 < clen) { - { - auto _gst_249 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_4; - } - { - auto _gst_250 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_4; - } - } - int t_5 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew_1052 + 256 + d_lane]; - float dkf_5 = pool_f32[ew_1052 + 1280 + d_lane]; - float dgf_5 = pool_f32[ew_1052 + 2304 + d_lane]; - float bt_t_5 = btv_1058[2]; - float e_off_5 = approx_exp2(g_t_5 - gn_1049); - float e_dg_5 = approx_exp2(g_t_5 - gc_1050); - float dq2_5 = __fmaf_rn(_tmem_load_12[2], e_off_5, _tmem_load_14[2] * e_dg_5); - float dk2_5 = __fmaf_rn(_tmem_load_13[2], e_off_5, _tmem_load_15[2] * e_dg_5); - dbp_1057[2] = dk2_5 * k_t_5; - float dk2b_5 = dk2_5 * bt_t_5; - float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_16[2]; - float dg_o_5 = __fmaf_rn(q_t_1_13, dq2_5, (dk2b_5 - _tmem_load_16[2]) * k_t_5) + dgf_5; - s_run_1059[0] = s_run_1059[0] + dg_o_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_5 = t_5 * qrs; - if (t_5 < clen) { - { - auto _gst_251 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_5; - } - { - auto _gst_252 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_5; - } - } - int t_6 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew_1052 + 128 + d_lane]; - float dkf_6 = pool_f32[ew_1052 + 1152 + d_lane]; - float dgf_6 = pool_f32[ew_1052 + 2176 + d_lane]; - float bt_t_6 = btv_1058[1]; - float e_off_6 = approx_exp2(g_t_6 - gn_1049); - float e_dg_6 = approx_exp2(g_t_6 - gc_1050); - float dq2_6 = __fmaf_rn(_tmem_load_12[1], e_off_6, _tmem_load_14[1] * e_dg_6); - float dk2_6 = __fmaf_rn(_tmem_load_13[1], e_off_6, _tmem_load_15[1] * e_dg_6); - dbp_1057[1] = dk2_6 * k_t_6; - float dk2b_6 = dk2_6 * bt_t_6; - float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_16[1]; - float dg_o_6 = __fmaf_rn(q_t_1_14, dq2_6, (dk2b_6 - _tmem_load_16[1]) * k_t_6) + dgf_6; - s_run_1059[0] = s_run_1059[0] + dg_o_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_6 = t_6 * qrs; - if (t_6 < clen) { - { - auto _gst_253 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_6; - } - { - auto _gst_254 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_6; - } - } - int t_7 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_1055 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_1056 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1053 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew_1052 + d_lane]; - float dkf_7 = pool_f32[ew_1052 + 1024 + d_lane]; - float dgf_7 = pool_f32[ew_1052 + 2048 + d_lane]; - float bt_t_7 = btv_1058[0]; - float e_off_7 = approx_exp2(g_t_7 - gn_1049); - float e_dg_7 = approx_exp2(g_t_7 - gc_1050); - float dq2_7 = __fmaf_rn(_tmem_load_12[0], e_off_7, _tmem_load_14[0] * e_dg_7); - float dk2_7 = __fmaf_rn(_tmem_load_13[0], e_off_7, _tmem_load_15[0] * e_dg_7); - dbp_1057[0] = dk2_7 * k_t_7; - float dk2b_7 = dk2_7 * bt_t_7; - float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_16[0]; - float dg_o_7 = __fmaf_rn(q_t_1_15, dq2_7, (dk2b_7 - _tmem_load_16[0]) * k_t_7) + dgf_7; - s_run_1059[0] = s_run_1059[0] + dg_o_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_1059[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_7 = t_7 * qrs; - if (t_7 < clen) { - { - auto _gst_255 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_7; - } - { - auto _gst_256 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_7; + float r_251[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_0 = r_251[0]; + float gc_1 = gn_0 - 21.640425f; + int eb_0 = 98304 + th * -32768; + int ew_0 = eb_0 >> 2; + int qbase_1 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); + int sb8_0 = 48 + 8 * th; + int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); + int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); + int c_qa_0 = 160 + 8 * th; + int c_ka_0 = 176 + 8 * th; + int c_qd_0 = 224 + 8 * th; + int c_kd_0 = 240 + 8 * th; + int c_dk_0 = 48 + 8 * th; + int c_bp_0 = c_ka_0; + float s_run_0[1]; + s_run_0[0] = 0.0f; + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_1 = 7 - k_1; + float a_qd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1))); + float a_kd_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_kd_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_kd_0 + j_1))); + float a_dk_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_dk_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_dk_0 + j_1))); + float a_qa_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_qa_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qa_0 + j_1))); + float a_ka_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(a_ka_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(c_ka_0 + j_1))); + float btr_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btr_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (sb8_0 + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_0 + j_1; + float r_252[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_252[0]; + unsigned int r_253[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_253[0] << 16); + unsigned int r_254[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1 = __uint_as_float(r_254[0] << 16); + float q_m_1 = ((t_1 >= clen) ? 0.0f : q_t_1); + float dqf_1 = pool_f32[ew_0 + 128 * j_1 + d_lane]; + float dkf_1 = pool_f32[ew_0 + (1024 + 128 * j_1) + d_lane]; + float dgf_1 = pool_f32[ew_0 + (2048 + 128 * j_1) + d_lane]; + float bt_t_1 = btr_1[0]; + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float e_off_1 = approx_exp2(g_t_1 - gn_0); + float dq2_1 = __fmaf_rn(a_qa_1[0], e_off_1, a_qd_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(a_ka_1[0], e_off_1, a_kd_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + a_dk_1[0]; + float qe_1 = q_m_1 * e_dg_1; + float kp_12 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_12; + float kbe_1 = (float)kb_1 * e_dg_1; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, a_qd_1[0], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_12 * e_dg_1) * a_kd_1[0]); + float dg_o_1 = __fmaf_rn(q_m_1, dq2_1, __fmaf_rn(dk2b_1 - a_dk_1[0], k_t_1, cr_1)) + dgf_1; + s_run_0[0] = s_run_0[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_qd_0 + j_1)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_1)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff_1 = t_1 * qrs; + if (t_1 < clen) { + { + auto _gst_259 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_1; + } + { + auto _gst_260 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_1; + } } } - float tot_1228 = s_run_1059[0]; - int b4_1229 = lane_1 >> 4 & 1; - int b3_1230 = lane_1 >> 3 & 1; - int b2_1231 = lane_1 >> 2 & 1; - float v4_1232[4]; - float send_0 = ((b4_1229 == 0) ? dbp_1057[4] : dbp_1057[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4_1232[0] = ((b4_1229 == 0) ? dbp_1057[0] : dbp_1057[4]) + _shfl_xor_sync_257; - float send_1 = ((b4_1229 == 0) ? dbp_1057[5] : dbp_1057[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4_1232[1] = ((b4_1229 == 0) ? dbp_1057[1] : dbp_1057[5]) + _shfl_xor_sync_258; - float send_2 = ((b4_1229 == 0) ? dbp_1057[6] : dbp_1057[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4_1232[2] = ((b4_1229 == 0) ? dbp_1057[2] : dbp_1057[6]) + _shfl_xor_sync_259; - float send_3 = ((b4_1229 == 0) ? dbp_1057[7] : dbp_1057[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4_1232[3] = ((b4_1229 == 0) ? dbp_1057[3] : dbp_1057[7]) + _shfl_xor_sync_260; - float v2_1237[2]; - float send_4 = ((b3_1230 == 0) ? v4_1232[2] : v4_1232[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2_1237[0] = ((b3_1230 == 0) ? v4_1232[0] : v4_1232[2]) + _shfl_xor_sync_261; - float send_5 = ((b3_1230 == 0) ? v4_1232[3] : v4_1232[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2_1237[1] = ((b3_1230 == 0) ? v4_1232[1] : v4_1232[3]) + _shfl_xor_sync_262; - float send_1240 = ((b2_1231 == 0) ? v2_1237[1] : v2_1237[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_1240, 4); - float v1_1241 = ((b2_1231 == 0) ? v2_1237[0] : v2_1237[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 2); - v1_1241 = v1_1241 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_1241, 1); - v1_1241 = v1_1241 + _shfl_xor_sync_265; + float tot_0 = s_run_0[0]; + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int b4_0 = lane_1 >> 4 & 1; + int b3_0 = lane_1 >> 3 & 1; + int b2_0 = lane_1 >> 2 & 1; + float v4_0[4]; + float send_4 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_4, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_261; + float send_5 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_5, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_262; + float send_6 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_263; + float send_7 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_264; + float v2_16[2]; + float send2_2 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send2_2, 8); + v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_265; + float send2_3 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send2_3, 8); + v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_266; + float send1_0 = ((b2_0 == 0) ? v2_16[1] : v2_16[0]); + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send1_0, 4); + float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_267; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_268; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_269; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); float gacc[8]; @@ -3803,8 +3388,8 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_1228) : "memory"); + uint32_t _addr_270 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_270), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3822,8 +3407,8 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.fence::after_thread_sync;"); float t_other[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); - float t_hi = ((th == 1) ? tot_1228 : t_other[0]); - float t_lo = ((th == 1) ? t_other[0] : tot_1228); + float t_hi = ((th == 1) ? tot_0 : t_other[0]); + float t_lo = ((th == 1) ? t_other[0] : tot_0); float offset = ((th == 1) ? off_acc[0] : off_acc[0] + t_hi); float sfx[8]; tmem_ld_x8(&sfx[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); @@ -3836,132 +3421,132 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_267 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_271 = make_float2(gacc[4], gacc[4]); float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&rp_u)); - float _abs_270; - asm("abs.f32 %0, %1;" : "=f"(_abs_270) : "f"(rp_z.x)); - float _neg_269; - asm("neg.f32 %0, %1;" : "=f"(_neg_269) : "f"(_abs_270)); - float _exp_268 = __nv_expf(_neg_269); - float _abs_273; - asm("abs.f32 %0, %1;" : "=f"(_abs_273) : "f"(rp_z.y)); - float _neg_272; - asm("neg.f32 %0, %1;" : "=f"(_neg_272) : "f"(_abs_273)); - float _exp_271 = __nv_expf(_neg_272); - float2 rp_e = make_float2(_exp_268, _exp_271); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_271), "l"(*(const unsigned long long*)&rp_u)); + float _abs_274; + asm("abs.f32 %0, %1;" : "=f"(_abs_274) : "f"(rp_z.x)); + float _neg_273; + asm("neg.f32 %0, %1;" : "=f"(_neg_273) : "f"(_abs_274)); + float _exp_272 = __nv_expf(_neg_273); + float _abs_277; + asm("abs.f32 %0, %1;" : "=f"(_abs_277) : "f"(rp_z.y)); + float _neg_276; + asm("neg.f32 %0, %1;" : "=f"(_neg_276) : "f"(_abs_277)); + float _exp_275 = __nv_expf(_neg_276); + float2 rp_e = make_float2(_exp_272, _exp_275); float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_274 = approx_rcp(rp_den.x); - float _rcp_275 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_276 = make_float2(_rcp_274, _rcp_275); + float _rcp_278 = approx_rcp(rp_den.x); + float _rcp_279 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_280 = make_float2(_rcp_278, _rcp_279); float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_276)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_280)); float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_277 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_281 = make_float2(gacc[5], gacc[5]); float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_281)); u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_278 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_282 = make_float2(gacc[4], gacc[4]); float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_278), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_281; - asm("abs.f32 %0, %1;" : "=f"(_abs_281) : "f"(rp_z_0.x)); - float _neg_280; - asm("neg.f32 %0, %1;" : "=f"(_neg_280) : "f"(_abs_281)); - float _exp_279 = __nv_expf(_neg_280); - float _abs_284; - asm("abs.f32 %0, %1;" : "=f"(_abs_284) : "f"(rp_z_0.y)); - float _neg_283; - asm("neg.f32 %0, %1;" : "=f"(_neg_283) : "f"(_abs_284)); - float _exp_282 = __nv_expf(_neg_283); - float2 rp_e_0 = make_float2(_exp_279, _exp_282); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_282), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_285; + asm("abs.f32 %0, %1;" : "=f"(_abs_285) : "f"(rp_z_0.x)); + float _neg_284; + asm("neg.f32 %0, %1;" : "=f"(_neg_284) : "f"(_abs_285)); + float _exp_283 = __nv_expf(_neg_284); + float _abs_288; + asm("abs.f32 %0, %1;" : "=f"(_abs_288) : "f"(rp_z_0.y)); + float _neg_287; + asm("neg.f32 %0, %1;" : "=f"(_neg_287) : "f"(_abs_288)); + float _exp_286 = __nv_expf(_neg_287); + float2 rp_e_0 = make_float2(_exp_283, _exp_286); float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_285 = approx_rcp(rp_den_0.x); - float _rcp_286 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_287 = make_float2(_rcp_285, _rcp_286); + float _rcp_289 = approx_rcp(rp_den_0.x); + float _rcp_290 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_291 = make_float2(_rcp_289, _rcp_290); float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_287)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_291)); float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_288 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_292 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_288)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_292)); u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_289 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_293 = make_float2(gacc[4], gacc[4]); float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_289), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_292; - asm("abs.f32 %0, %1;" : "=f"(_abs_292) : "f"(rp_z_1.x)); - float _neg_291; - asm("neg.f32 %0, %1;" : "=f"(_neg_291) : "f"(_abs_292)); - float _exp_290 = __nv_expf(_neg_291); - float _abs_295; - asm("abs.f32 %0, %1;" : "=f"(_abs_295) : "f"(rp_z_1.y)); - float _neg_294; - asm("neg.f32 %0, %1;" : "=f"(_neg_294) : "f"(_abs_295)); - float _exp_293 = __nv_expf(_neg_294); - float2 rp_e_1 = make_float2(_exp_290, _exp_293); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_293), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_296; + asm("abs.f32 %0, %1;" : "=f"(_abs_296) : "f"(rp_z_1.x)); + float _neg_295; + asm("neg.f32 %0, %1;" : "=f"(_neg_295) : "f"(_abs_296)); + float _exp_294 = __nv_expf(_neg_295); + float _abs_299; + asm("abs.f32 %0, %1;" : "=f"(_abs_299) : "f"(rp_z_1.y)); + float _neg_298; + asm("neg.f32 %0, %1;" : "=f"(_neg_298) : "f"(_abs_299)); + float _exp_297 = __nv_expf(_neg_298); + float2 rp_e_1 = make_float2(_exp_294, _exp_297); float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_296 = approx_rcp(rp_den_1.x); - float _rcp_297 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_298 = make_float2(_rcp_296, _rcp_297); + float _rcp_300 = approx_rcp(rp_den_1.x); + float _rcp_301 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_302 = make_float2(_rcp_300, _rcp_301); float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_298)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_302)); float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_299 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_303 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_299)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_303)); u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_300 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_304 = make_float2(gacc[4], gacc[4]); float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_303; - asm("abs.f32 %0, %1;" : "=f"(_abs_303) : "f"(rp_z_2.x)); - float _neg_302; - asm("neg.f32 %0, %1;" : "=f"(_neg_302) : "f"(_abs_303)); - float _exp_301 = __nv_expf(_neg_302); - float _abs_306; - asm("abs.f32 %0, %1;" : "=f"(_abs_306) : "f"(rp_z_2.y)); - float _neg_305; - asm("neg.f32 %0, %1;" : "=f"(_neg_305) : "f"(_abs_306)); - float _exp_304 = __nv_expf(_neg_305); - float2 rp_e_2 = make_float2(_exp_301, _exp_304); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_304), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_307; + asm("abs.f32 %0, %1;" : "=f"(_abs_307) : "f"(rp_z_2.x)); + float _neg_306; + asm("neg.f32 %0, %1;" : "=f"(_neg_306) : "f"(_abs_307)); + float _exp_305 = __nv_expf(_neg_306); + float _abs_310; + asm("abs.f32 %0, %1;" : "=f"(_abs_310) : "f"(rp_z_2.y)); + float _neg_309; + asm("neg.f32 %0, %1;" : "=f"(_neg_309) : "f"(_abs_310)); + float _exp_308 = __nv_expf(_neg_309); + float2 rp_e_2 = make_float2(_exp_305, _exp_308); float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_307 = approx_rcp(rp_den_2.x); - float _rcp_308 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_309 = make_float2(_rcp_307, _rcp_308); + float _rcp_311 = approx_rcp(rp_den_2.x); + float _rcp_312 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_313 = make_float2(_rcp_311, _rcp_312); float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_313)); float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_310 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_314 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_310)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_314)); u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -3976,54 +3561,54 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_311; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_311) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_311; + uint32_t _prmt_315; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_315; int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_312 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = pq_0[0]; + auto _gst_316 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = pq_0[0]; } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_313; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_313) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_313; + uint32_t _prmt_317; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_317; int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_314 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = pq_1[0]; + auto _gst_318 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = pq_1[0]; } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_315; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_315; + uint32_t _prmt_319; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_319) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_319; int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_316 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = pq_2[0]; + auto _gst_320 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = pq_2[0]; } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_317; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_317; + uint32_t _prmt_321; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_321) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_321; int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_318 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = pq_3[0]; + auto _gst_322 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast(_gst_322) + (0)) = pq_3[0]; } } int t_a_0 = 48 + 8 * th; @@ -4088,47 +3673,47 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_319; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_319) : "f"(s_a), "f"(nm_t)); + float _sub_323; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_323) : "f"(s_a), "f"(nm_t)); float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_319), "f"(pa)); - float _sub_320; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_320) : "f"(pa), "f"(nm_t)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_323), "f"(pa)); + float _sub_324; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_324) : "f"(pa), "f"(nm_t)); float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_320), "f"(s_a)); - float _abs_322; - asm("abs.f32 %0, %1;" : "=f"(_abs_322) : "f"(s_a)); - float _abs_323; - asm("abs.f32 %0, %1;" : "=f"(_abs_323) : "f"(pa)); - uint32_t _setp_321_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_321_bits) : "f"(_abs_322), "f"(_abs_323)); - bool _setp_321 = _setp_321_bits != 0; - float nm_err = ((_setp_321) ? nm_es : nm_ex); - float _add_324; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(c_a), "f"(nm_err)); - c_a = _add_324; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_324), "f"(s_a)); + float _abs_326; + asm("abs.f32 %0, %1;" : "=f"(_abs_326) : "f"(s_a)); + float _abs_327; + asm("abs.f32 %0, %1;" : "=f"(_abs_327) : "f"(pa)); + uint32_t _setp_325_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_325_bits) : "f"(_abs_326), "f"(_abs_327)); + bool _setp_325 = _setp_325_bits != 0; + float nm_err = ((_setp_325) ? nm_es : nm_ex); + float _add_328; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(c_a), "f"(nm_err)); + c_a = _add_328; s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_325; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_325) : "f"(s_b), "f"(nm_t_0)); + float _sub_329; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_329) : "f"(s_b), "f"(nm_t_0)); float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_325), "f"(pb)); - float _sub_326; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_326) : "f"(pb), "f"(nm_t_0)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_329), "f"(pb)); + float _sub_330; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_330) : "f"(pb), "f"(nm_t_0)); float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_326), "f"(s_b)); - float _abs_328; - asm("abs.f32 %0, %1;" : "=f"(_abs_328) : "f"(s_b)); - float _abs_329; - asm("abs.f32 %0, %1;" : "=f"(_abs_329) : "f"(pb)); - uint32_t _setp_327_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_327_bits) : "f"(_abs_328), "f"(_abs_329)); - bool _setp_327 = _setp_327_bits != 0; - float nm_err_0 = ((_setp_327) ? nm_es_0 : nm_ex_0); - float _add_330; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_330; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_330), "f"(s_b)); + float _abs_332; + asm("abs.f32 %0, %1;" : "=f"(_abs_332) : "f"(s_b)); + float _abs_333; + asm("abs.f32 %0, %1;" : "=f"(_abs_333) : "f"(pb)); + uint32_t _setp_331_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_331_bits) : "f"(_abs_332), "f"(_abs_333)); + bool _setp_331 = _setp_331_bits != 0; + float nm_err_0 = ((_setp_331) ? nm_es_0 : nm_ex_0); + float _add_334; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_334; s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4152,132 +3737,132 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_331 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_335 = make_float2(gacc[4], gacc[4]); float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_331), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_334; - asm("abs.f32 %0, %1;" : "=f"(_abs_334) : "f"(rp_z_3.x)); - float _neg_333; - asm("neg.f32 %0, %1;" : "=f"(_neg_333) : "f"(_abs_334)); - float _exp_332 = __nv_expf(_neg_333); - float _abs_337; - asm("abs.f32 %0, %1;" : "=f"(_abs_337) : "f"(rp_z_3.y)); - float _neg_336; - asm("neg.f32 %0, %1;" : "=f"(_neg_336) : "f"(_abs_337)); - float _exp_335 = __nv_expf(_neg_336); - float2 rp_e_3 = make_float2(_exp_332, _exp_335); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_335), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_338; + asm("abs.f32 %0, %1;" : "=f"(_abs_338) : "f"(rp_z_3.x)); + float _neg_337; + asm("neg.f32 %0, %1;" : "=f"(_neg_337) : "f"(_abs_338)); + float _exp_336 = __nv_expf(_neg_337); + float _abs_341; + asm("abs.f32 %0, %1;" : "=f"(_abs_341) : "f"(rp_z_3.y)); + float _neg_340; + asm("neg.f32 %0, %1;" : "=f"(_neg_340) : "f"(_abs_341)); + float _exp_339 = __nv_expf(_neg_340); + float2 rp_e_3 = make_float2(_exp_336, _exp_339); float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_338 = approx_rcp(rp_den_3.x); - float _rcp_339 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_340 = make_float2(_rcp_338, _rcp_339); + float _rcp_342 = approx_rcp(rp_den_3.x); + float _rcp_343 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_344 = make_float2(_rcp_342, _rcp_343); float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_340)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_344)); float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_341 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_345 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_345)); u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_342 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_346 = make_float2(gacc[4], gacc[4]); float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_342), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_345; - asm("abs.f32 %0, %1;" : "=f"(_abs_345) : "f"(rp_z_4.x)); - float _neg_344; - asm("neg.f32 %0, %1;" : "=f"(_neg_344) : "f"(_abs_345)); - float _exp_343 = __nv_expf(_neg_344); - float _abs_348; - asm("abs.f32 %0, %1;" : "=f"(_abs_348) : "f"(rp_z_4.y)); - float _neg_347; - asm("neg.f32 %0, %1;" : "=f"(_neg_347) : "f"(_abs_348)); - float _exp_346 = __nv_expf(_neg_347); - float2 rp_e_4 = make_float2(_exp_343, _exp_346); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_346), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_349; + asm("abs.f32 %0, %1;" : "=f"(_abs_349) : "f"(rp_z_4.x)); + float _neg_348; + asm("neg.f32 %0, %1;" : "=f"(_neg_348) : "f"(_abs_349)); + float _exp_347 = __nv_expf(_neg_348); + float _abs_352; + asm("abs.f32 %0, %1;" : "=f"(_abs_352) : "f"(rp_z_4.y)); + float _neg_351; + asm("neg.f32 %0, %1;" : "=f"(_neg_351) : "f"(_abs_352)); + float _exp_350 = __nv_expf(_neg_351); + float2 rp_e_4 = make_float2(_exp_347, _exp_350); float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_349 = approx_rcp(rp_den_4.x); - float _rcp_350 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_351 = make_float2(_rcp_349, _rcp_350); + float _rcp_353 = approx_rcp(rp_den_4.x); + float _rcp_354 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_355 = make_float2(_rcp_353, _rcp_354); float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_351)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_355)); float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_352 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_356 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_352)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_356)); u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_353 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_357 = make_float2(gacc[4], gacc[4]); float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_356; - asm("abs.f32 %0, %1;" : "=f"(_abs_356) : "f"(rp_z_5.x)); - float _neg_355; - asm("neg.f32 %0, %1;" : "=f"(_neg_355) : "f"(_abs_356)); - float _exp_354 = __nv_expf(_neg_355); - float _abs_359; - asm("abs.f32 %0, %1;" : "=f"(_abs_359) : "f"(rp_z_5.y)); - float _neg_358; - asm("neg.f32 %0, %1;" : "=f"(_neg_358) : "f"(_abs_359)); - float _exp_357 = __nv_expf(_neg_358); - float2 rp_e_5 = make_float2(_exp_354, _exp_357); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_357), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_360; + asm("abs.f32 %0, %1;" : "=f"(_abs_360) : "f"(rp_z_5.x)); + float _neg_359; + asm("neg.f32 %0, %1;" : "=f"(_neg_359) : "f"(_abs_360)); + float _exp_358 = __nv_expf(_neg_359); + float _abs_363; + asm("abs.f32 %0, %1;" : "=f"(_abs_363) : "f"(rp_z_5.y)); + float _neg_362; + asm("neg.f32 %0, %1;" : "=f"(_neg_362) : "f"(_abs_363)); + float _exp_361 = __nv_expf(_neg_362); + float2 rp_e_5 = make_float2(_exp_358, _exp_361); float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_360 = approx_rcp(rp_den_5.x); - float _rcp_361 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_362 = make_float2(_rcp_360, _rcp_361); + float _rcp_364 = approx_rcp(rp_den_5.x); + float _rcp_365 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_366 = make_float2(_rcp_364, _rcp_365); float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_362)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_366)); float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_363 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_367 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_363)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_367)); u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_364 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_368 = make_float2(gacc[4], gacc[4]); float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_364), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_367; - asm("abs.f32 %0, %1;" : "=f"(_abs_367) : "f"(rp_z_6.x)); - float _neg_366; - asm("neg.f32 %0, %1;" : "=f"(_neg_366) : "f"(_abs_367)); - float _exp_365 = __nv_expf(_neg_366); - float _abs_370; - asm("abs.f32 %0, %1;" : "=f"(_abs_370) : "f"(rp_z_6.y)); - float _neg_369; - asm("neg.f32 %0, %1;" : "=f"(_neg_369) : "f"(_abs_370)); - float _exp_368 = __nv_expf(_neg_369); - float2 rp_e_6 = make_float2(_exp_365, _exp_368); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_368), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_371; + asm("abs.f32 %0, %1;" : "=f"(_abs_371) : "f"(rp_z_6.x)); + float _neg_370; + asm("neg.f32 %0, %1;" : "=f"(_neg_370) : "f"(_abs_371)); + float _exp_369 = __nv_expf(_neg_370); + float _abs_374; + asm("abs.f32 %0, %1;" : "=f"(_abs_374) : "f"(rp_z_6.y)); + float _neg_373; + asm("neg.f32 %0, %1;" : "=f"(_neg_373) : "f"(_abs_374)); + float _exp_372 = __nv_expf(_neg_373); + float2 rp_e_6 = make_float2(_exp_369, _exp_372); float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_371 = approx_rcp(rp_den_6.x); - float _rcp_372 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_373 = make_float2(_rcp_371, _rcp_372); + float _rcp_375 = approx_rcp(rp_den_6.x); + float _rcp_376 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_377 = make_float2(_rcp_375, _rcp_376); float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_373)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_377)); float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_374 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_378 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_374)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_378)); u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4292,54 +3877,54 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_375; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_375) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_375; + uint32_t _prmt_379; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_379; int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_376 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_376)); - *(reinterpret_cast(_gst_376) + (0)) = pq_4[0]; + auto _gst_380 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = pq_4[0]; } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_377; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_377) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_377; + uint32_t _prmt_381; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_381; int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_378 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_378)); - *(reinterpret_cast(_gst_378) + (0)) = pq_5[0]; + auto _gst_382 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = pq_5[0]; } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_379; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_379; + uint32_t _prmt_383; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_383) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_383; int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_380 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_380)); - *(reinterpret_cast(_gst_380) + (0)) = pq_6[0]; + auto _gst_384 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = pq_6[0]; } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_381; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_381; + uint32_t _prmt_385; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_385) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_385; int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_382 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_382)); - *(reinterpret_cast(_gst_382) + (0)) = pq_7[0]; + auto _gst_386 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_386)); + *(reinterpret_cast(_gst_386) + (0)) = pq_7[0]; } } int t_a_8 = 32 + 8 * th; @@ -4404,47 +3989,47 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_383; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_383) : "f"(s_a_0), "f"(nm_t_1)); + float _sub_387; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_387) : "f"(s_a_0), "f"(nm_t_1)); float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_383), "f"(pa_0)); - float _sub_384; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_384) : "f"(pa_0), "f"(nm_t_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_387), "f"(pa_0)); + float _sub_388; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_388) : "f"(pa_0), "f"(nm_t_1)); float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_384), "f"(s_a_0)); - float _abs_386; - asm("abs.f32 %0, %1;" : "=f"(_abs_386) : "f"(s_a_0)); - float _abs_387; - asm("abs.f32 %0, %1;" : "=f"(_abs_387) : "f"(pa_0)); - uint32_t _setp_385_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_385_bits) : "f"(_abs_386), "f"(_abs_387)); - bool _setp_385 = _setp_385_bits != 0; - float nm_err_1 = ((_setp_385) ? nm_es_1 : nm_ex_1); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_388), "f"(s_a_0)); + float _abs_390; + asm("abs.f32 %0, %1;" : "=f"(_abs_390) : "f"(s_a_0)); + float _abs_391; + asm("abs.f32 %0, %1;" : "=f"(_abs_391) : "f"(pa_0)); + uint32_t _setp_389_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_389_bits) : "f"(_abs_390), "f"(_abs_391)); + bool _setp_389 = _setp_389_bits != 0; + float nm_err_1 = ((_setp_389) ? nm_es_1 : nm_ex_1); + float _add_392; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_392; s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_389; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_389) : "f"(s_b_0), "f"(nm_t_2)); + float _sub_393; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_393) : "f"(s_b_0), "f"(nm_t_2)); float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_389), "f"(pb_0)); - float _sub_390; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_390) : "f"(pb_0), "f"(nm_t_2)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_393), "f"(pb_0)); + float _sub_394; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_394) : "f"(pb_0), "f"(nm_t_2)); float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_390), "f"(s_b_0)); - float _abs_392; - asm("abs.f32 %0, %1;" : "=f"(_abs_392) : "f"(s_b_0)); - float _abs_393; - asm("abs.f32 %0, %1;" : "=f"(_abs_393) : "f"(pb_0)); - uint32_t _setp_391_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_391_bits) : "f"(_abs_392), "f"(_abs_393)); - bool _setp_391 = _setp_391_bits != 0; - float nm_err_2 = ((_setp_391) ? nm_es_2 : nm_ex_2); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_394; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_394), "f"(s_b_0)); + float _abs_396; + asm("abs.f32 %0, %1;" : "=f"(_abs_396) : "f"(s_b_0)); + float _abs_397; + asm("abs.f32 %0, %1;" : "=f"(_abs_397) : "f"(pb_0)); + uint32_t _setp_395_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_395_bits) : "f"(_abs_396), "f"(_abs_397)); + bool _setp_395 = _setp_395_bits != 0; + float nm_err_2 = ((_setp_395) ? nm_es_2 : nm_ex_2); + float _add_398; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_398; s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4452,8 +4037,8 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_395 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_395), "f"(off_acc[0]) : "memory"); + uint32_t _addr_399 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_399), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4479,9 +4064,9 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1292 = r_297[0]; + float r_255[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1292 = r_255[0]; float gc_1293 = gn_1292 - 21.640425f; int eb_1294 = 81920 + th * -32768; int ew_1295 = eb_1294 >> 2; @@ -4510,348 +4095,388 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1302[1]; s_run_1302[0] = 0.0f; - int t_8 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_1295 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_1295 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_1295 + 2944 + d_lane]; - float bt_t_8 = btv_1301[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_1292); - float e_dg_8 = approx_exp2(g_t_8 - gc_1293); - float dq2_8 = __fmaf_rn(_tmem_load_20[7], e_off_8, _tmem_load_22[7] * e_dg_8); - float dk2_8 = __fmaf_rn(_tmem_load_21[7], e_off_8, _tmem_load_23[7] * e_dg_8); - dbp_1300[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_24[7]; - float dg_o_8 = __fmaf_rn(q_t_1_16, dq2_8, (dk2b_8 - _tmem_load_24[7]) * k_t_8) + dgf_8; - s_run_1302[0] = s_run_1302[0] + dg_o_8; + int t_0 = 16 + 8 * th + 7; + float r_256[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_0 = r_256[0]; + unsigned int r_257[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_0 = __uint_as_float(r_257[0] << 16); + unsigned int r_258[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_0 = __uint_as_float(r_258[0] << 16); + float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); + float dqf_0 = pool_f32[ew_1295 + 896 + d_lane]; + float dkf_0 = pool_f32[ew_1295 + 1920 + d_lane]; + float dgf_0 = pool_f32[ew_1295 + 2944 + d_lane]; + float bt_t_0 = btv_1301[7]; + float e_off_0 = approx_exp2(g_t_0 - gn_1292); + float e_dg_0 = approx_exp2(g_t_0 - gc_1293); + float dq2_0 = __fmaf_rn(_tmem_load_20[7], e_off_0, _tmem_load_22[7] * e_dg_0); + float dk2_0 = __fmaf_rn(_tmem_load_21[7], e_off_0, _tmem_load_23[7] * e_dg_0); + dbp_1300[7] = dk2_0 * k_t_0; + float dk2b_0 = dk2_0 * bt_t_0; + float dq_o_0 = dq2_0 + dqf_0; + float dk_o_0 = dk2b_0 + dkf_0 + _tmem_load_24[7]; + float qe_2 = q_t_1_0 * e_dg_0; + float kp_13 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_13; + float kbe_2 = (float)kb_2 * e_dg_0; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, _tmem_load_22[7], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_13 * e_dg_0) * _tmem_load_23[7]); + float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, __fmaf_rn(dk2b_0 - _tmem_load_24[7], k_t_0, cr_2)) + dgf_0; + s_run_1302[0] = s_run_1302[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_8 = t_8 * qrs; - if (t_8 < clen) { + int qoff_0 = t_0 * qrs; + if (t_0 < clen) { { - auto _gst_396 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_396)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_396) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_400 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_400)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_397 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_397)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_397) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_401 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_401)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_0; } } - int t_9 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_1295 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_1295 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_1295 + 2816 + d_lane]; - float bt_t_9 = btv_1301[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_1292); - float e_dg_9 = approx_exp2(g_t_9 - gc_1293); - float dq2_9 = __fmaf_rn(_tmem_load_20[6], e_off_9, _tmem_load_22[6] * e_dg_9); - float dk2_9 = __fmaf_rn(_tmem_load_21[6], e_off_9, _tmem_load_23[6] * e_dg_9); - dbp_1300[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_24[6]; - float dg_o_9 = __fmaf_rn(q_t_1_17, dq2_9, (dk2b_9 - _tmem_load_24[6]) * k_t_9) + dgf_9; - s_run_1302[0] = s_run_1302[0] + dg_o_9; + int t_1_1 = 16 + 8 * th + 6; + float r_259[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1_1 = r_259[0]; + unsigned int r_260[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1_1 = __uint_as_float(r_260[0] << 16); + unsigned int r_261[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1_1 = __uint_as_float(r_261[0] << 16); + float q_t_1_1_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_1); + float dqf_1_1 = pool_f32[ew_1295 + 768 + d_lane]; + float dkf_1_1 = pool_f32[ew_1295 + 1792 + d_lane]; + float dgf_1_1 = pool_f32[ew_1295 + 2816 + d_lane]; + float bt_t_1_1 = btv_1301[6]; + float e_off_1_1 = approx_exp2(g_t_1_1 - gn_1292); + float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_1293); + float dq2_1_1 = __fmaf_rn(_tmem_load_20[6], e_off_1_1, _tmem_load_22[6] * e_dg_1_1); + float dk2_1_1 = __fmaf_rn(_tmem_load_21[6], e_off_1_1, _tmem_load_23[6] * e_dg_1_1); + dbp_1300[6] = dk2_1_1 * k_t_1_1; + float dk2b_1_1 = dk2_1_1 * bt_t_1_1; + float dq_o_1_1 = dq2_1_1 + dqf_1_1; + float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + _tmem_load_24[6]; + float qe_0 = q_t_1_1_1 * e_dg_1_1; + float kp_12_1 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12_1; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, _tmem_load_22[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12_1 * e_dg_1_1) * _tmem_load_23[6]); + float dg_o_1_1 = __fmaf_rn(q_t_1_1_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - _tmem_load_24[6], k_t_1_1, cr_0)) + dgf_1_1; + s_run_1302[0] = s_run_1302[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_9 = t_9 * qrs; - if (t_9 < clen) { + int qoff_1_1 = t_1_1 * qrs; + if (t_1_1 < clen) { { - auto _gst_398 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_398)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_398) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_402 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_402)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_1_1; } { - auto _gst_399 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_399)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_399) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_403 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_403)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_1_1; } } - int t_10 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_1295 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_1295 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_1295 + 2688 + d_lane]; - float bt_t_10 = btv_1301[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_1292); - float e_dg_10 = approx_exp2(g_t_10 - gc_1293); - float dq2_10 = __fmaf_rn(_tmem_load_20[5], e_off_10, _tmem_load_22[5] * e_dg_10); - float dk2_10 = __fmaf_rn(_tmem_load_21[5], e_off_10, _tmem_load_23[5] * e_dg_10); - dbp_1300[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_24[5]; - float dg_o_10 = __fmaf_rn(q_t_1_18, dq2_10, (dk2b_10 - _tmem_load_24[5]) * k_t_10) + dgf_10; - s_run_1302[0] = s_run_1302[0] + dg_o_10; + int t_2 = 16 + 8 * th + 5; + float r_262[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_262[0]; + unsigned int r_263[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_263[0] << 16); + unsigned int r_264[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_264[0] << 16); + float q_t_1_2 = ((t_2 >= clen) ? 0.0f : q_t_2); + float dqf_2 = pool_f32[ew_1295 + 640 + d_lane]; + float dkf_2 = pool_f32[ew_1295 + 1664 + d_lane]; + float dgf_2 = pool_f32[ew_1295 + 2688 + d_lane]; + float bt_t_2 = btv_1301[5]; + float e_off_2 = approx_exp2(g_t_2 - gn_1292); + float e_dg_2 = approx_exp2(g_t_2 - gc_1293); + float dq2_2 = __fmaf_rn(_tmem_load_20[5], e_off_2, _tmem_load_22[5] * e_dg_2); + float dk2_2 = __fmaf_rn(_tmem_load_21[5], e_off_2, _tmem_load_23[5] * e_dg_2); + dbp_1300[5] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + _tmem_load_24[5]; + float qe_1_1 = q_t_1_2 * e_dg_2; + float kp_13_1 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1_1 = (__nv_bfloat16)kp_13_1; + float kbe_1_1 = (float)kb_1_1 * e_dg_2; + float cr_1_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1_1) & 4294959104u) - qe_1_1, _tmem_load_22[5], (__uint_as_float(__as_u32(kbe_1_1) & 4294959104u) - kp_13_1 * e_dg_2) * _tmem_load_23[5]); + float dg_o_2 = __fmaf_rn(q_t_1_2, dq2_2, __fmaf_rn(dk2b_2 - _tmem_load_24[5], k_t_2, cr_1_1)) + dgf_2; + s_run_1302[0] = s_run_1302[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_10 = t_10 * qrs; - if (t_10 < clen) { + int qoff_2 = t_2 * qrs; + if (t_2 < clen) { { - auto _gst_400 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_400)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_404 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_404)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_401 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_401)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_405 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_405)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_2; } } - int t_11 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_1295 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_1295 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_1295 + 2560 + d_lane]; - float bt_t_11 = btv_1301[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_1292); - float e_dg_11 = approx_exp2(g_t_11 - gc_1293); - float dq2_11 = __fmaf_rn(_tmem_load_20[4], e_off_11, _tmem_load_22[4] * e_dg_11); - float dk2_11 = __fmaf_rn(_tmem_load_21[4], e_off_11, _tmem_load_23[4] * e_dg_11); - dbp_1300[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_24[4]; - float dg_o_11 = __fmaf_rn(q_t_1_19, dq2_11, (dk2b_11 - _tmem_load_24[4]) * k_t_11) + dgf_11; - s_run_1302[0] = s_run_1302[0] + dg_o_11; + int t_3 = 16 + 8 * th + 4; + float r_265[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_3 = r_265[0]; + unsigned int r_266[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_3 = __uint_as_float(r_266[0] << 16); + unsigned int r_267[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_3 = __uint_as_float(r_267[0] << 16); + float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); + float dqf_3 = pool_f32[ew_1295 + 512 + d_lane]; + float dkf_3 = pool_f32[ew_1295 + 1536 + d_lane]; + float dgf_3 = pool_f32[ew_1295 + 2560 + d_lane]; + float bt_t_3 = btv_1301[4]; + float e_off_3 = approx_exp2(g_t_3 - gn_1292); + float e_dg_3 = approx_exp2(g_t_3 - gc_1293); + float dq2_3 = __fmaf_rn(_tmem_load_20[4], e_off_3, _tmem_load_22[4] * e_dg_3); + float dk2_3 = __fmaf_rn(_tmem_load_21[4], e_off_3, _tmem_load_23[4] * e_dg_3); + dbp_1300[4] = dk2_3 * k_t_3; + float dk2b_3 = dk2_3 * bt_t_3; + float dq_o_3 = dq2_3 + dqf_3; + float dk_o_3 = dk2b_3 + dkf_3 + _tmem_load_24[4]; + float qe_2_1 = q_t_1_3 * e_dg_3; + float kp_14 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2_1 = (__nv_bfloat16)kp_14; + float kbe_2_1 = (float)kb_2_1 * e_dg_3; + float cr_2_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_2_1) & 4294959104u) - qe_2_1, _tmem_load_22[4], (__uint_as_float(__as_u32(kbe_2_1) & 4294959104u) - kp_14 * e_dg_3) * _tmem_load_23[4]); + float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, __fmaf_rn(dk2b_3 - _tmem_load_24[4], k_t_3, cr_2_1)) + dgf_3; + s_run_1302[0] = s_run_1302[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_11 = t_11 * qrs; - if (t_11 < clen) { + int qoff_3 = t_3 * qrs; + if (t_3 < clen) { { - auto _gst_402 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_402)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_406 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_406)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_403 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_403)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_407 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_407)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_3; } } - int t_12 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_1295 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_1295 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_1295 + 2432 + d_lane]; - float bt_t_12 = btv_1301[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_1292); - float e_dg_12 = approx_exp2(g_t_12 - gc_1293); - float dq2_12 = __fmaf_rn(_tmem_load_20[3], e_off_12, _tmem_load_22[3] * e_dg_12); - float dk2_12 = __fmaf_rn(_tmem_load_21[3], e_off_12, _tmem_load_23[3] * e_dg_12); - dbp_1300[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_24[3]; - float dg_o_12 = __fmaf_rn(q_t_1_20, dq2_12, (dk2b_12 - _tmem_load_24[3]) * k_t_12) + dgf_12; - s_run_1302[0] = s_run_1302[0] + dg_o_12; + int t_4 = 16 + 8 * th + 3; + float r_268[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_4 = r_268[0]; + unsigned int r_269[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_4 = __uint_as_float(r_269[0] << 16); + unsigned int r_270[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_4 = __uint_as_float(r_270[0] << 16); + float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); + float dqf_4 = pool_f32[ew_1295 + 384 + d_lane]; + float dkf_4 = pool_f32[ew_1295 + 1408 + d_lane]; + float dgf_4 = pool_f32[ew_1295 + 2432 + d_lane]; + float bt_t_4 = btv_1301[3]; + float e_off_4 = approx_exp2(g_t_4 - gn_1292); + float e_dg_4 = approx_exp2(g_t_4 - gc_1293); + float dq2_4 = __fmaf_rn(_tmem_load_20[3], e_off_4, _tmem_load_22[3] * e_dg_4); + float dk2_4 = __fmaf_rn(_tmem_load_21[3], e_off_4, _tmem_load_23[3] * e_dg_4); + dbp_1300[3] = dk2_4 * k_t_4; + float dk2b_4 = dk2_4 * bt_t_4; + float dq_o_4 = dq2_4 + dqf_4; + float dk_o_4 = dk2b_4 + dkf_4 + _tmem_load_24[3]; + float qe_3 = q_t_1_4 * e_dg_4; + float kp_15 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, _tmem_load_22[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_4) * _tmem_load_23[3]); + float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, __fmaf_rn(dk2b_4 - _tmem_load_24[3], k_t_4, cr_3)) + dgf_4; + s_run_1302[0] = s_run_1302[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_12 = t_12 * qrs; - if (t_12 < clen) { + int qoff_4 = t_4 * qrs; + if (t_4 < clen) { { - auto _gst_404 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_404)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_408 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_408)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_405 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_405)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_409 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_409)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_4; } } - int t_13 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_1295 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_1295 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_1295 + 2304 + d_lane]; - float bt_t_13 = btv_1301[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_1292); - float e_dg_13 = approx_exp2(g_t_13 - gc_1293); - float dq2_13 = __fmaf_rn(_tmem_load_20[2], e_off_13, _tmem_load_22[2] * e_dg_13); - float dk2_13 = __fmaf_rn(_tmem_load_21[2], e_off_13, _tmem_load_23[2] * e_dg_13); - dbp_1300[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_24[2]; - float dg_o_13 = __fmaf_rn(q_t_1_21, dq2_13, (dk2b_13 - _tmem_load_24[2]) * k_t_13) + dgf_13; - s_run_1302[0] = s_run_1302[0] + dg_o_13; + int t_5 = 16 + 8 * th + 2; + float r_271[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_5 = r_271[0]; + unsigned int r_272[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_5 = __uint_as_float(r_272[0] << 16); + unsigned int r_273[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_5 = __uint_as_float(r_273[0] << 16); + float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); + float dqf_5 = pool_f32[ew_1295 + 256 + d_lane]; + float dkf_5 = pool_f32[ew_1295 + 1280 + d_lane]; + float dgf_5 = pool_f32[ew_1295 + 2304 + d_lane]; + float bt_t_5 = btv_1301[2]; + float e_off_5 = approx_exp2(g_t_5 - gn_1292); + float e_dg_5 = approx_exp2(g_t_5 - gc_1293); + float dq2_5 = __fmaf_rn(_tmem_load_20[2], e_off_5, _tmem_load_22[2] * e_dg_5); + float dk2_5 = __fmaf_rn(_tmem_load_21[2], e_off_5, _tmem_load_23[2] * e_dg_5); + dbp_1300[2] = dk2_5 * k_t_5; + float dk2b_5 = dk2_5 * bt_t_5; + float dq_o_5 = dq2_5 + dqf_5; + float dk_o_5 = dk2b_5 + dkf_5 + _tmem_load_24[2]; + float qe_4 = q_t_1_5 * e_dg_5; + float kp_16 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, _tmem_load_22[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_5) * _tmem_load_23[2]); + float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, __fmaf_rn(dk2b_5 - _tmem_load_24[2], k_t_5, cr_4)) + dgf_5; + s_run_1302[0] = s_run_1302[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_13 = t_13 * qrs; - if (t_13 < clen) { + int qoff_5 = t_5 * qrs; + if (t_5 < clen) { { - auto _gst_406 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_406)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_410 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_410)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_407 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_407)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_411 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_411)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_5; } } - int t_14 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_1295 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_1295 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_1295 + 2176 + d_lane]; - float bt_t_14 = btv_1301[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_1292); - float e_dg_14 = approx_exp2(g_t_14 - gc_1293); - float dq2_14 = __fmaf_rn(_tmem_load_20[1], e_off_14, _tmem_load_22[1] * e_dg_14); - float dk2_14 = __fmaf_rn(_tmem_load_21[1], e_off_14, _tmem_load_23[1] * e_dg_14); - dbp_1300[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_24[1]; - float dg_o_14 = __fmaf_rn(q_t_1_22, dq2_14, (dk2b_14 - _tmem_load_24[1]) * k_t_14) + dgf_14; - s_run_1302[0] = s_run_1302[0] + dg_o_14; + int t_6 = 16 + 8 * th + 1; + float r_274[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_6 = r_274[0]; + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_6 = __uint_as_float(r_275[0] << 16); + unsigned int r_276[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_6 = __uint_as_float(r_276[0] << 16); + float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); + float dqf_6 = pool_f32[ew_1295 + 128 + d_lane]; + float dkf_6 = pool_f32[ew_1295 + 1152 + d_lane]; + float dgf_6 = pool_f32[ew_1295 + 2176 + d_lane]; + float bt_t_6 = btv_1301[1]; + float e_off_6 = approx_exp2(g_t_6 - gn_1292); + float e_dg_6 = approx_exp2(g_t_6 - gc_1293); + float dq2_6 = __fmaf_rn(_tmem_load_20[1], e_off_6, _tmem_load_22[1] * e_dg_6); + float dk2_6 = __fmaf_rn(_tmem_load_21[1], e_off_6, _tmem_load_23[1] * e_dg_6); + dbp_1300[1] = dk2_6 * k_t_6; + float dk2b_6 = dk2_6 * bt_t_6; + float dq_o_6 = dq2_6 + dqf_6; + float dk_o_6 = dk2b_6 + dkf_6 + _tmem_load_24[1]; + float qe_5 = q_t_1_6 * e_dg_6; + float kp_17 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, _tmem_load_22[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_6) * _tmem_load_23[1]); + float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, __fmaf_rn(dk2b_6 - _tmem_load_24[1], k_t_6, cr_5)) + dgf_6; + s_run_1302[0] = s_run_1302[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_14 = t_14 * qrs; - if (t_14 < clen) { + int qoff_6 = t_6 * qrs; + if (t_6 < clen) { { - auto _gst_408 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_408)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_412 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_412)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_412) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_409 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_409)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_413 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_413)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_413) + (0)) = (__nv_bfloat16)dk_o_6; } } - int t_15 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_1295 + d_lane]; - float dkf_15 = pool_f32[ew_1295 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_1295 + 2048 + d_lane]; - float bt_t_15 = btv_1301[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_1292); - float e_dg_15 = approx_exp2(g_t_15 - gc_1293); - float dq2_15 = __fmaf_rn(_tmem_load_20[0], e_off_15, _tmem_load_22[0] * e_dg_15); - float dk2_15 = __fmaf_rn(_tmem_load_21[0], e_off_15, _tmem_load_23[0] * e_dg_15); - dbp_1300[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_24[0]; - float dg_o_15 = __fmaf_rn(q_t_1_23, dq2_15, (dk2b_15 - _tmem_load_24[0]) * k_t_15) + dgf_15; - s_run_1302[0] = s_run_1302[0] + dg_o_15; + int t_7 = 16 + 8 * th; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1298 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_7 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1299 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_7 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_1296 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_7 = __uint_as_float(r_279[0] << 16); + float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); + float dqf_7 = pool_f32[ew_1295 + d_lane]; + float dkf_7 = pool_f32[ew_1295 + 1024 + d_lane]; + float dgf_7 = pool_f32[ew_1295 + 2048 + d_lane]; + float bt_t_7 = btv_1301[0]; + float e_off_7 = approx_exp2(g_t_7 - gn_1292); + float e_dg_7 = approx_exp2(g_t_7 - gc_1293); + float dq2_7 = __fmaf_rn(_tmem_load_20[0], e_off_7, _tmem_load_22[0] * e_dg_7); + float dk2_7 = __fmaf_rn(_tmem_load_21[0], e_off_7, _tmem_load_23[0] * e_dg_7); + dbp_1300[0] = dk2_7 * k_t_7; + float dk2b_7 = dk2_7 * bt_t_7; + float dq_o_7 = dq2_7 + dqf_7; + float dk_o_7 = dk2b_7 + dkf_7 + _tmem_load_24[0]; + float qe_6 = q_t_1_7 * e_dg_7; + float kp_18 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, _tmem_load_22[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_7) * _tmem_load_23[0]); + float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, __fmaf_rn(dk2b_7 - _tmem_load_24[0], k_t_7, cr_6)) + dgf_7; + s_run_1302[0] = s_run_1302[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1302[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_15 = t_15 * qrs; - if (t_15 < clen) { + int qoff_7 = t_7 * qrs; + if (t_7 < clen) { { - auto _gst_410 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_410)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_414 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_414)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_414) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_411 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_411)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_415 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_415)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_415) + (0)) = (__nv_bfloat16)dk_o_7; } } float tot_1471 = s_run_1302[0]; @@ -4859,40 +4484,40 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d int b3_1473 = lane_1 >> 3 & 1; int b2_1474 = lane_1 >> 2 & 1; float v4_1475[4]; - float send_6 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); - float _shfl_xor_sync_412 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_412; - float send_7 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); - float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_413; - float send_8 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); - float _shfl_xor_sync_414 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_414; - float send_9 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); - float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_415; + float send_8 = ((b4_1472 == 0) ? dbp_1300[4] : dbp_1300[0]); + float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_1475[0] = ((b4_1472 == 0) ? dbp_1300[0] : dbp_1300[4]) + _shfl_xor_sync_416; + float send_9 = ((b4_1472 == 0) ? dbp_1300[5] : dbp_1300[1]); + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_1475[1] = ((b4_1472 == 0) ? dbp_1300[1] : dbp_1300[5]) + _shfl_xor_sync_417; + float send_10 = ((b4_1472 == 0) ? dbp_1300[6] : dbp_1300[2]); + float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_10, 16); + v4_1475[2] = ((b4_1472 == 0) ? dbp_1300[2] : dbp_1300[6]) + _shfl_xor_sync_418; + float send_11 = ((b4_1472 == 0) ? dbp_1300[7] : dbp_1300[3]); + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, send_11, 16); + v4_1475[3] = ((b4_1472 == 0) ? dbp_1300[3] : dbp_1300[7]) + _shfl_xor_sync_419; float v2_1480[2]; - float send_10 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); - float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_416; - float send_11 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); - float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_417; + float send_12 = ((b3_1473 == 0) ? v4_1475[2] : v4_1475[0]); + float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, send_12, 8); + v2_1480[0] = ((b3_1473 == 0) ? v4_1475[0] : v4_1475[2]) + _shfl_xor_sync_420; + float send_13 = ((b3_1473 == 0) ? v4_1475[3] : v4_1475[1]); + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, send_13, 8); + v2_1480[1] = ((b3_1473 == 0) ? v4_1475[1] : v4_1475[3]) + _shfl_xor_sync_421; float send_1483 = ((b2_1474 == 0) ? v2_1480[1] : v2_1480[0]); - float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); - float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_418; - float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); - v1_1484 = v1_1484 + _shfl_xor_sync_419; - float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); - v1_1484 = v1_1484 + _shfl_xor_sync_420; + float _shfl_xor_sync_422 = __shfl_xor_sync(0xFFFFFFFF, send_1483, 4); + float v1_1484 = ((b2_1474 == 0) ? v2_1480[0] : v2_1480[1]) + _shfl_xor_sync_422; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 2); + v1_1484 = v1_1484 + _shfl_xor_sync_423; + float _shfl_xor_sync_424 = __shfl_xor_sync(0xFFFFFFFF, v1_1484, 1); + v1_1484 = v1_1484 + _shfl_xor_sync_424; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_421 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_421), "f"(tot_1471) : "memory"); + uint32_t _addr_425 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_425), "f"(tot_1471) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4924,132 +4549,132 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_422 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_426 = make_float2(gacc[4], gacc[4]); float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_422), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_425; - asm("abs.f32 %0, %1;" : "=f"(_abs_425) : "f"(rp_z_7.x)); - float _neg_424; - asm("neg.f32 %0, %1;" : "=f"(_neg_424) : "f"(_abs_425)); - float _exp_423 = __nv_expf(_neg_424); - float _abs_428; - asm("abs.f32 %0, %1;" : "=f"(_abs_428) : "f"(rp_z_7.y)); - float _neg_427; - asm("neg.f32 %0, %1;" : "=f"(_neg_427) : "f"(_abs_428)); - float _exp_426 = __nv_expf(_neg_427); - float2 rp_e_7 = make_float2(_exp_423, _exp_426); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_426), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_429; + asm("abs.f32 %0, %1;" : "=f"(_abs_429) : "f"(rp_z_7.x)); + float _neg_428; + asm("neg.f32 %0, %1;" : "=f"(_neg_428) : "f"(_abs_429)); + float _exp_427 = __nv_expf(_neg_428); + float _abs_432; + asm("abs.f32 %0, %1;" : "=f"(_abs_432) : "f"(rp_z_7.y)); + float _neg_431; + asm("neg.f32 %0, %1;" : "=f"(_neg_431) : "f"(_abs_432)); + float _exp_430 = __nv_expf(_neg_431); + float2 rp_e_7 = make_float2(_exp_427, _exp_430); float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_429 = approx_rcp(rp_den_7.x); - float _rcp_430 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_431 = make_float2(_rcp_429, _rcp_430); + float _rcp_433 = approx_rcp(rp_den_7.x); + float _rcp_434 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_435 = make_float2(_rcp_433, _rcp_434); float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_431)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_435)); float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_432 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_436 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_432)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_436)); u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_433 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_437 = make_float2(gacc[4], gacc[4]); float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_433), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_436; - asm("abs.f32 %0, %1;" : "=f"(_abs_436) : "f"(rp_z_8.x)); - float _neg_435; - asm("neg.f32 %0, %1;" : "=f"(_neg_435) : "f"(_abs_436)); - float _exp_434 = __nv_expf(_neg_435); - float _abs_439; - asm("abs.f32 %0, %1;" : "=f"(_abs_439) : "f"(rp_z_8.y)); - float _neg_438; - asm("neg.f32 %0, %1;" : "=f"(_neg_438) : "f"(_abs_439)); - float _exp_437 = __nv_expf(_neg_438); - float2 rp_e_8 = make_float2(_exp_434, _exp_437); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_437), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_440; + asm("abs.f32 %0, %1;" : "=f"(_abs_440) : "f"(rp_z_8.x)); + float _neg_439; + asm("neg.f32 %0, %1;" : "=f"(_neg_439) : "f"(_abs_440)); + float _exp_438 = __nv_expf(_neg_439); + float _abs_443; + asm("abs.f32 %0, %1;" : "=f"(_abs_443) : "f"(rp_z_8.y)); + float _neg_442; + asm("neg.f32 %0, %1;" : "=f"(_neg_442) : "f"(_abs_443)); + float _exp_441 = __nv_expf(_neg_442); + float2 rp_e_8 = make_float2(_exp_438, _exp_441); float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_440 = approx_rcp(rp_den_8.x); - float _rcp_441 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_442 = make_float2(_rcp_440, _rcp_441); + float _rcp_444 = approx_rcp(rp_den_8.x); + float _rcp_445 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_446 = make_float2(_rcp_444, _rcp_445); float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_442)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_446)); float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_443 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_447 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_443)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_447)); u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_444 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_448 = make_float2(gacc[4], gacc[4]); float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_444), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_447; - asm("abs.f32 %0, %1;" : "=f"(_abs_447) : "f"(rp_z_9.x)); - float _neg_446; - asm("neg.f32 %0, %1;" : "=f"(_neg_446) : "f"(_abs_447)); - float _exp_445 = __nv_expf(_neg_446); - float _abs_450; - asm("abs.f32 %0, %1;" : "=f"(_abs_450) : "f"(rp_z_9.y)); - float _neg_449; - asm("neg.f32 %0, %1;" : "=f"(_neg_449) : "f"(_abs_450)); - float _exp_448 = __nv_expf(_neg_449); - float2 rp_e_9 = make_float2(_exp_445, _exp_448); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_448), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_451; + asm("abs.f32 %0, %1;" : "=f"(_abs_451) : "f"(rp_z_9.x)); + float _neg_450; + asm("neg.f32 %0, %1;" : "=f"(_neg_450) : "f"(_abs_451)); + float _exp_449 = __nv_expf(_neg_450); + float _abs_454; + asm("abs.f32 %0, %1;" : "=f"(_abs_454) : "f"(rp_z_9.y)); + float _neg_453; + asm("neg.f32 %0, %1;" : "=f"(_neg_453) : "f"(_abs_454)); + float _exp_452 = __nv_expf(_neg_453); + float2 rp_e_9 = make_float2(_exp_449, _exp_452); float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_451 = approx_rcp(rp_den_9.x); - float _rcp_452 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_453 = make_float2(_rcp_451, _rcp_452); + float _rcp_455 = approx_rcp(rp_den_9.x); + float _rcp_456 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_457 = make_float2(_rcp_455, _rcp_456); float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_453)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_457)); float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_454 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_458 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_454)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_455 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_459 = make_float2(gacc[4], gacc[4]); float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_455), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_458; - asm("abs.f32 %0, %1;" : "=f"(_abs_458) : "f"(rp_z_10.x)); - float _neg_457; - asm("neg.f32 %0, %1;" : "=f"(_neg_457) : "f"(_abs_458)); - float _exp_456 = __nv_expf(_neg_457); - float _abs_461; - asm("abs.f32 %0, %1;" : "=f"(_abs_461) : "f"(rp_z_10.y)); - float _neg_460; - asm("neg.f32 %0, %1;" : "=f"(_neg_460) : "f"(_abs_461)); - float _exp_459 = __nv_expf(_neg_460); - float2 rp_e_10 = make_float2(_exp_456, _exp_459); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_459), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_462; + asm("abs.f32 %0, %1;" : "=f"(_abs_462) : "f"(rp_z_10.x)); + float _neg_461; + asm("neg.f32 %0, %1;" : "=f"(_neg_461) : "f"(_abs_462)); + float _exp_460 = __nv_expf(_neg_461); + float _abs_465; + asm("abs.f32 %0, %1;" : "=f"(_abs_465) : "f"(rp_z_10.y)); + float _neg_464; + asm("neg.f32 %0, %1;" : "=f"(_neg_464) : "f"(_abs_465)); + float _exp_463 = __nv_expf(_neg_464); + float2 rp_e_10 = make_float2(_exp_460, _exp_463); float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_462 = approx_rcp(rp_den_10.x); - float _rcp_463 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_464 = make_float2(_rcp_462, _rcp_463); + float _rcp_466 = approx_rcp(rp_den_10.x); + float _rcp_467 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_468 = make_float2(_rcp_466, _rcp_467); float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_464)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_468)); float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_465 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_469 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_465)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -5064,54 +4689,54 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_466; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_466) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_466; + uint32_t _prmt_470; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_470; int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_467 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_467)); - *(reinterpret_cast(_gst_467) + (0)) = pq_8[0]; + auto _gst_471 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_471)); + *(reinterpret_cast(_gst_471) + (0)) = pq_8[0]; } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_468; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_468) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_468; + uint32_t _prmt_472; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_472; int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_469 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_469)); - *(reinterpret_cast(_gst_469) + (0)) = pq_9[0]; + auto _gst_473 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_473)); + *(reinterpret_cast(_gst_473) + (0)) = pq_9[0]; } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_470; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_470; + uint32_t _prmt_474; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_474) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_474; int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_471 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_471)); - *(reinterpret_cast(_gst_471) + (0)) = pq_10[0]; + auto _gst_475 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_475)); + *(reinterpret_cast(_gst_475) + (0)) = pq_10[0]; } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_472; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_472; + uint32_t _prmt_476; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_476) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_476; int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_473 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_473)); - *(reinterpret_cast(_gst_473) + (0)) = pq_11[0]; + auto _gst_477 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_477)); + *(reinterpret_cast(_gst_477) + (0)) = pq_11[0]; } } int t_a_16 = 16 + 8 * th; @@ -5176,47 +4801,47 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_474; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_474) : "f"(s_a_1), "f"(nm_t_3)); + float _sub_478; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_478) : "f"(s_a_1), "f"(nm_t_3)); float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_474), "f"(pa_1)); - float _sub_475; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_475) : "f"(pa_1), "f"(nm_t_3)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_478), "f"(pa_1)); + float _sub_479; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_479) : "f"(pa_1), "f"(nm_t_3)); float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_475), "f"(s_a_1)); - float _abs_477; - asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(s_a_1)); - float _abs_478; - asm("abs.f32 %0, %1;" : "=f"(_abs_478) : "f"(pa_1)); - uint32_t _setp_476_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_476_bits) : "f"(_abs_477), "f"(_abs_478)); - bool _setp_476 = _setp_476_bits != 0; - float nm_err_3 = ((_setp_476) ? nm_es_3 : nm_ex_3); - float _add_479; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_479) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_479; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_479), "f"(s_a_1)); + float _abs_481; + asm("abs.f32 %0, %1;" : "=f"(_abs_481) : "f"(s_a_1)); + float _abs_482; + asm("abs.f32 %0, %1;" : "=f"(_abs_482) : "f"(pa_1)); + uint32_t _setp_480_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_480_bits) : "f"(_abs_481), "f"(_abs_482)); + bool _setp_480 = _setp_480_bits != 0; + float nm_err_3 = ((_setp_480) ? nm_es_3 : nm_ex_3); + float _add_483; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_483) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_483; s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_480; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_480) : "f"(s_b_1), "f"(nm_t_4)); + float _sub_484; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_484) : "f"(s_b_1), "f"(nm_t_4)); float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_480), "f"(pb_1)); - float _sub_481; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_481) : "f"(pb_1), "f"(nm_t_4)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_484), "f"(pb_1)); + float _sub_485; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_485) : "f"(pb_1), "f"(nm_t_4)); float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_481), "f"(s_b_1)); - float _abs_483; - asm("abs.f32 %0, %1;" : "=f"(_abs_483) : "f"(s_b_1)); - float _abs_484; - asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(pb_1)); - uint32_t _setp_482_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_482_bits) : "f"(_abs_483), "f"(_abs_484)); - bool _setp_482 = _setp_482_bits != 0; - float nm_err_4 = ((_setp_482) ? nm_es_4 : nm_ex_4); - float _add_485; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_485) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_485; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_485), "f"(s_b_1)); + float _abs_487; + asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(s_b_1)); + float _abs_488; + asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(pb_1)); + uint32_t _setp_486_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_486_bits) : "f"(_abs_487), "f"(_abs_488)); + bool _setp_486 = _setp_486_bits != 0; + float nm_err_4 = ((_setp_486) ? nm_es_4 : nm_ex_4); + float _add_489; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_489) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_489; s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5224,8 +4849,8 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_486 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_486), "f"(off_acc[0]) : "memory"); + uint32_t _addr_490 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_490), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5249,9 +4874,9 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1514 = r_322[0]; + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1514 = r_280[0]; float gc_1515 = gn_1514 - 21.640425f; int eb_1516 = 98304 + th * -32768; int ew_1517 = eb_1516 >> 2; @@ -5276,348 +4901,388 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1524[1]; s_run_1524[0] = 0.0f; - int t_16 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1517 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1517 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1517 + 2944 + d_lane]; - float bt_t_16 = btv_1523[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1514); - float e_dg_16 = approx_exp2(g_t_16 - gc_1515); - float dq2_16 = _tmem_load_26[7] * e_dg_16; - float dk2_16 = _tmem_load_27[7] * e_dg_16; - dbp_1522[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + _tmem_load_28[7]; - float dg_o_16 = __fmaf_rn(q_t_1_24, dq2_16, (dk2b_16 - _tmem_load_28[7]) * k_t_16) + dgf_16; - s_run_1524[0] = s_run_1524[0] + dg_o_16; + int t_8 = 8 * th + 7; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_8 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_8 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_8 = __uint_as_float(r_283[0] << 16); + float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); + float dqf_8 = pool_f32[ew_1517 + 896 + d_lane]; + float dkf_8 = pool_f32[ew_1517 + 1920 + d_lane]; + float dgf_8 = pool_f32[ew_1517 + 2944 + d_lane]; + float bt_t_8 = btv_1523[7]; + float e_off_8 = approx_exp2(g_t_8 - gn_1514); + float e_dg_8 = approx_exp2(g_t_8 - gc_1515); + float dq2_8 = _tmem_load_26[7] * e_dg_8; + float dk2_8 = _tmem_load_27[7] * e_dg_8; + dbp_1522[7] = dk2_8 * k_t_8; + float dk2b_8 = dk2_8 * bt_t_8; + float dq_o_8 = dq2_8 + dqf_8; + float dk_o_8 = dk2b_8 + dkf_8 + _tmem_load_28[7]; + float qe_7 = q_t_1_8 * e_dg_8; + float kp_19 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, _tmem_load_26[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_8) * _tmem_load_27[7]); + float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, __fmaf_rn(dk2b_8 - _tmem_load_28[7], k_t_8, cr_7)) + dgf_8; + s_run_1524[0] = s_run_1524[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_16 = t_16 * qrs; - if (t_16 < clen) { + int qoff_8 = t_8 * qrs; + if (t_8 < clen) { { - auto _gst_487 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_487)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_487) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_491 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_491)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_488 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_488)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_488) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_492 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_492)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_8; } } - int t_17 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1517 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1517 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1517 + 2816 + d_lane]; - float bt_t_17 = btv_1523[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1514); - float e_dg_17 = approx_exp2(g_t_17 - gc_1515); - float dq2_17 = _tmem_load_26[6] * e_dg_17; - float dk2_17 = _tmem_load_27[6] * e_dg_17; - dbp_1522[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + _tmem_load_28[6]; - float dg_o_17 = __fmaf_rn(q_t_1_25, dq2_17, (dk2b_17 - _tmem_load_28[6]) * k_t_17) + dgf_17; - s_run_1524[0] = s_run_1524[0] + dg_o_17; + int t_9 = 8 * th + 6; + float r_284[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_9 = r_284[0]; + unsigned int r_285[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_9 = __uint_as_float(r_285[0] << 16); + unsigned int r_286[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_9 = __uint_as_float(r_286[0] << 16); + float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); + float dqf_9 = pool_f32[ew_1517 + 768 + d_lane]; + float dkf_9 = pool_f32[ew_1517 + 1792 + d_lane]; + float dgf_9 = pool_f32[ew_1517 + 2816 + d_lane]; + float bt_t_9 = btv_1523[6]; + float e_off_9 = approx_exp2(g_t_9 - gn_1514); + float e_dg_9 = approx_exp2(g_t_9 - gc_1515); + float dq2_9 = _tmem_load_26[6] * e_dg_9; + float dk2_9 = _tmem_load_27[6] * e_dg_9; + dbp_1522[6] = dk2_9 * k_t_9; + float dk2b_9 = dk2_9 * bt_t_9; + float dq_o_9 = dq2_9 + dqf_9; + float dk_o_9 = dk2b_9 + dkf_9 + _tmem_load_28[6]; + float qe_8 = q_t_1_9 * e_dg_9; + float kp_20 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, _tmem_load_26[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_9) * _tmem_load_27[6]); + float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, __fmaf_rn(dk2b_9 - _tmem_load_28[6], k_t_9, cr_8)) + dgf_9; + s_run_1524[0] = s_run_1524[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_17 = t_17 * qrs; - if (t_17 < clen) { + int qoff_9 = t_9 * qrs; + if (t_9 < clen) { { - auto _gst_489 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_489)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_489) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_493 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_493)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_490 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_490)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_490) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_494 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_494)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_9; } } - int t_18 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1517 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1517 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1517 + 2688 + d_lane]; - float bt_t_18 = btv_1523[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1514); - float e_dg_18 = approx_exp2(g_t_18 - gc_1515); - float dq2_18 = _tmem_load_26[5] * e_dg_18; - float dk2_18 = _tmem_load_27[5] * e_dg_18; - dbp_1522[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + _tmem_load_28[5]; - float dg_o_18 = __fmaf_rn(q_t_1_26, dq2_18, (dk2b_18 - _tmem_load_28[5]) * k_t_18) + dgf_18; - s_run_1524[0] = s_run_1524[0] + dg_o_18; + int t_10 = 8 * th + 5; + float r_287[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_10 = r_287[0]; + unsigned int r_288[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_10 = __uint_as_float(r_288[0] << 16); + unsigned int r_289[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_10 = __uint_as_float(r_289[0] << 16); + float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); + float dqf_10 = pool_f32[ew_1517 + 640 + d_lane]; + float dkf_10 = pool_f32[ew_1517 + 1664 + d_lane]; + float dgf_10 = pool_f32[ew_1517 + 2688 + d_lane]; + float bt_t_10 = btv_1523[5]; + float e_off_10 = approx_exp2(g_t_10 - gn_1514); + float e_dg_10 = approx_exp2(g_t_10 - gc_1515); + float dq2_10 = _tmem_load_26[5] * e_dg_10; + float dk2_10 = _tmem_load_27[5] * e_dg_10; + dbp_1522[5] = dk2_10 * k_t_10; + float dk2b_10 = dk2_10 * bt_t_10; + float dq_o_10 = dq2_10 + dqf_10; + float dk_o_10 = dk2b_10 + dkf_10 + _tmem_load_28[5]; + float qe_9 = q_t_1_10 * e_dg_10; + float kp_21 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, _tmem_load_26[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_10) * _tmem_load_27[5]); + float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, __fmaf_rn(dk2b_10 - _tmem_load_28[5], k_t_10, cr_9)) + dgf_10; + s_run_1524[0] = s_run_1524[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_18 = t_18 * qrs; - if (t_18 < clen) { + int qoff_10 = t_10 * qrs; + if (t_10 < clen) { { - auto _gst_491 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_491)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_495 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_495)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_492 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_492)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_496 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_496)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_10; } } - int t_19 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1517 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1517 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1517 + 2560 + d_lane]; - float bt_t_19 = btv_1523[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1514); - float e_dg_19 = approx_exp2(g_t_19 - gc_1515); - float dq2_19 = _tmem_load_26[4] * e_dg_19; - float dk2_19 = _tmem_load_27[4] * e_dg_19; - dbp_1522[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + _tmem_load_28[4]; - float dg_o_19 = __fmaf_rn(q_t_1_27, dq2_19, (dk2b_19 - _tmem_load_28[4]) * k_t_19) + dgf_19; - s_run_1524[0] = s_run_1524[0] + dg_o_19; + int t_11 = 8 * th + 4; + float r_290[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_11 = r_290[0]; + unsigned int r_291[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_11 = __uint_as_float(r_291[0] << 16); + unsigned int r_292[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_11 = __uint_as_float(r_292[0] << 16); + float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); + float dqf_11 = pool_f32[ew_1517 + 512 + d_lane]; + float dkf_11 = pool_f32[ew_1517 + 1536 + d_lane]; + float dgf_11 = pool_f32[ew_1517 + 2560 + d_lane]; + float bt_t_11 = btv_1523[4]; + float e_off_11 = approx_exp2(g_t_11 - gn_1514); + float e_dg_11 = approx_exp2(g_t_11 - gc_1515); + float dq2_11 = _tmem_load_26[4] * e_dg_11; + float dk2_11 = _tmem_load_27[4] * e_dg_11; + dbp_1522[4] = dk2_11 * k_t_11; + float dk2b_11 = dk2_11 * bt_t_11; + float dq_o_11 = dq2_11 + dqf_11; + float dk_o_11 = dk2b_11 + dkf_11 + _tmem_load_28[4]; + float qe_10 = q_t_1_11 * e_dg_11; + float kp_22 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, _tmem_load_26[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_11) * _tmem_load_27[4]); + float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, __fmaf_rn(dk2b_11 - _tmem_load_28[4], k_t_11, cr_10)) + dgf_11; + s_run_1524[0] = s_run_1524[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_19 = t_19 * qrs; - if (t_19 < clen) { + int qoff_11 = t_11 * qrs; + if (t_11 < clen) { { - auto _gst_493 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_493)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_497 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_497)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_494 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_494)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_498 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_498)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_11; } } - int t_20 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1517 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1517 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1517 + 2432 + d_lane]; - float bt_t_20 = btv_1523[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1514); - float e_dg_20 = approx_exp2(g_t_20 - gc_1515); - float dq2_20 = _tmem_load_26[3] * e_dg_20; - float dk2_20 = _tmem_load_27[3] * e_dg_20; - dbp_1522[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + _tmem_load_28[3]; - float dg_o_20 = __fmaf_rn(q_t_1_28, dq2_20, (dk2b_20 - _tmem_load_28[3]) * k_t_20) + dgf_20; - s_run_1524[0] = s_run_1524[0] + dg_o_20; + int t_12 = 8 * th + 3; + float r_293[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_12 = r_293[0]; + unsigned int r_294[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_12 = __uint_as_float(r_294[0] << 16); + unsigned int r_295[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_12 = __uint_as_float(r_295[0] << 16); + float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); + float dqf_12 = pool_f32[ew_1517 + 384 + d_lane]; + float dkf_12 = pool_f32[ew_1517 + 1408 + d_lane]; + float dgf_12 = pool_f32[ew_1517 + 2432 + d_lane]; + float bt_t_12 = btv_1523[3]; + float e_off_12 = approx_exp2(g_t_12 - gn_1514); + float e_dg_12 = approx_exp2(g_t_12 - gc_1515); + float dq2_12 = _tmem_load_26[3] * e_dg_12; + float dk2_12 = _tmem_load_27[3] * e_dg_12; + dbp_1522[3] = dk2_12 * k_t_12; + float dk2b_12 = dk2_12 * bt_t_12; + float dq_o_12 = dq2_12 + dqf_12; + float dk_o_12 = dk2b_12 + dkf_12 + _tmem_load_28[3]; + float qe_11 = q_t_1_12 * e_dg_12; + float kp_23 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, _tmem_load_26[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_12) * _tmem_load_27[3]); + float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, __fmaf_rn(dk2b_12 - _tmem_load_28[3], k_t_12, cr_11)) + dgf_12; + s_run_1524[0] = s_run_1524[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_20 = t_20 * qrs; - if (t_20 < clen) { + int qoff_12 = t_12 * qrs; + if (t_12 < clen) { { - auto _gst_495 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_495)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_499 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_499)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_496 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_496)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_500 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_500)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_12; } } - int t_21 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1517 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1517 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1517 + 2304 + d_lane]; - float bt_t_21 = btv_1523[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1514); - float e_dg_21 = approx_exp2(g_t_21 - gc_1515); - float dq2_21 = _tmem_load_26[2] * e_dg_21; - float dk2_21 = _tmem_load_27[2] * e_dg_21; - dbp_1522[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + _tmem_load_28[2]; - float dg_o_21 = __fmaf_rn(q_t_1_29, dq2_21, (dk2b_21 - _tmem_load_28[2]) * k_t_21) + dgf_21; - s_run_1524[0] = s_run_1524[0] + dg_o_21; + int t_13 = 8 * th + 2; + float r_296[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_13 = r_296[0]; + unsigned int r_297[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_13 = __uint_as_float(r_297[0] << 16); + unsigned int r_298[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_13 = __uint_as_float(r_298[0] << 16); + float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); + float dqf_13 = pool_f32[ew_1517 + 256 + d_lane]; + float dkf_13 = pool_f32[ew_1517 + 1280 + d_lane]; + float dgf_13 = pool_f32[ew_1517 + 2304 + d_lane]; + float bt_t_13 = btv_1523[2]; + float e_off_13 = approx_exp2(g_t_13 - gn_1514); + float e_dg_13 = approx_exp2(g_t_13 - gc_1515); + float dq2_13 = _tmem_load_26[2] * e_dg_13; + float dk2_13 = _tmem_load_27[2] * e_dg_13; + dbp_1522[2] = dk2_13 * k_t_13; + float dk2b_13 = dk2_13 * bt_t_13; + float dq_o_13 = dq2_13 + dqf_13; + float dk_o_13 = dk2b_13 + dkf_13 + _tmem_load_28[2]; + float qe_12 = q_t_1_13 * e_dg_13; + float kp_24 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, _tmem_load_26[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_13) * _tmem_load_27[2]); + float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, __fmaf_rn(dk2b_13 - _tmem_load_28[2], k_t_13, cr_12)) + dgf_13; + s_run_1524[0] = s_run_1524[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_21 = t_21 * qrs; - if (t_21 < clen) { + int qoff_13 = t_13 * qrs; + if (t_13 < clen) { { - auto _gst_497 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_497)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_501 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_501)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_498 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_498)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_502 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_502)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_13; } } - int t_22 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1517 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1517 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1517 + 2176 + d_lane]; - float bt_t_22 = btv_1523[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1514); - float e_dg_22 = approx_exp2(g_t_22 - gc_1515); - float dq2_22 = _tmem_load_26[1] * e_dg_22; - float dk2_22 = _tmem_load_27[1] * e_dg_22; - dbp_1522[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + _tmem_load_28[1]; - float dg_o_22 = __fmaf_rn(q_t_1_30, dq2_22, (dk2b_22 - _tmem_load_28[1]) * k_t_22) + dgf_22; - s_run_1524[0] = s_run_1524[0] + dg_o_22; + int t_14 = 8 * th + 1; + float r_299[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_14 = r_299[0]; + unsigned int r_300[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_14 = __uint_as_float(r_300[0] << 16); + unsigned int r_301[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_14 = __uint_as_float(r_301[0] << 16); + float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); + float dqf_14 = pool_f32[ew_1517 + 128 + d_lane]; + float dkf_14 = pool_f32[ew_1517 + 1152 + d_lane]; + float dgf_14 = pool_f32[ew_1517 + 2176 + d_lane]; + float bt_t_14 = btv_1523[1]; + float e_off_14 = approx_exp2(g_t_14 - gn_1514); + float e_dg_14 = approx_exp2(g_t_14 - gc_1515); + float dq2_14 = _tmem_load_26[1] * e_dg_14; + float dk2_14 = _tmem_load_27[1] * e_dg_14; + dbp_1522[1] = dk2_14 * k_t_14; + float dk2b_14 = dk2_14 * bt_t_14; + float dq_o_14 = dq2_14 + dqf_14; + float dk_o_14 = dk2b_14 + dkf_14 + _tmem_load_28[1]; + float qe_13 = q_t_1_14 * e_dg_14; + float kp_25 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, _tmem_load_26[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_14) * _tmem_load_27[1]); + float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, __fmaf_rn(dk2b_14 - _tmem_load_28[1], k_t_14, cr_13)) + dgf_14; + s_run_1524[0] = s_run_1524[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_22 = t_22 * qrs; - if (t_22 < clen) { + int qoff_14 = t_14 * qrs; + if (t_14 < clen) { { - auto _gst_499 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_499)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_503 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_503)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_503) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_500 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_500)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_504 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_504)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_504) + (0)) = (__nv_bfloat16)dk_o_14; } } - int t_23 = 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1517 + d_lane]; - float dkf_23 = pool_f32[ew_1517 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1517 + 2048 + d_lane]; - float bt_t_23 = btv_1523[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1514); - float e_dg_23 = approx_exp2(g_t_23 - gc_1515); - float dq2_23 = _tmem_load_26[0] * e_dg_23; - float dk2_23 = _tmem_load_27[0] * e_dg_23; - dbp_1522[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + _tmem_load_28[0]; - float dg_o_23 = __fmaf_rn(q_t_1_31, dq2_23, (dk2b_23 - _tmem_load_28[0]) * k_t_23) + dgf_23; - s_run_1524[0] = s_run_1524[0] + dg_o_23; + int t_15 = 8 * th; + float r_302[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((gb8_1520 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_15 = r_302[0]; + unsigned int r_303[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((kb8_1521 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_15 = __uint_as_float(r_303[0] << 16); + unsigned int r_304[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((qbase_1518 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_15 = __uint_as_float(r_304[0] << 16); + float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); + float dqf_15 = pool_f32[ew_1517 + d_lane]; + float dkf_15 = pool_f32[ew_1517 + 1024 + d_lane]; + float dgf_15 = pool_f32[ew_1517 + 2048 + d_lane]; + float bt_t_15 = btv_1523[0]; + float e_off_15 = approx_exp2(g_t_15 - gn_1514); + float e_dg_15 = approx_exp2(g_t_15 - gc_1515); + float dq2_15 = _tmem_load_26[0] * e_dg_15; + float dk2_15 = _tmem_load_27[0] * e_dg_15; + dbp_1522[0] = dk2_15 * k_t_15; + float dk2b_15 = dk2_15 * bt_t_15; + float dq_o_15 = dq2_15 + dqf_15; + float dk_o_15 = dk2b_15 + dkf_15 + _tmem_load_28[0]; + float qe_14 = q_t_1_15 * e_dg_15; + float kp_26 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, _tmem_load_26[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_15) * _tmem_load_27[0]); + float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, __fmaf_rn(dk2b_15 - _tmem_load_28[0], k_t_15, cr_14)) + dgf_15; + s_run_1524[0] = s_run_1524[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_1524[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_23 = t_23 * qrs; - if (t_23 < clen) { + int qoff_15 = t_15 * qrs; + if (t_15 < clen) { { - auto _gst_501 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_501)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_505 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_505)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_505) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_502 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_502)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_506 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_506)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_506) + (0)) = (__nv_bfloat16)dk_o_15; } } float tot_1693 = s_run_1524[0]; @@ -5625,38 +5290,38 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d int b3_1695 = lane_1 >> 3 & 1; int b2_1696 = lane_1 >> 2 & 1; float v4_1697[4]; - float send_12 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); - float _shfl_xor_sync_503 = __shfl_xor_sync(0xFFFFFFFF, send_12, 16); - v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_503; - float send_13 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); - float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_504; - float send_14 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); - float _shfl_xor_sync_505 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_505; - float send_15 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); - float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_506; + float send_14 = ((b4_1694 == 0) ? dbp_1522[4] : dbp_1522[0]); + float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1697[0] = ((b4_1694 == 0) ? dbp_1522[0] : dbp_1522[4]) + _shfl_xor_sync_507; + float send_15 = ((b4_1694 == 0) ? dbp_1522[5] : dbp_1522[1]); + float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1697[1] = ((b4_1694 == 0) ? dbp_1522[1] : dbp_1522[5]) + _shfl_xor_sync_508; + float send_16 = ((b4_1694 == 0) ? dbp_1522[6] : dbp_1522[2]); + float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1697[2] = ((b4_1694 == 0) ? dbp_1522[2] : dbp_1522[6]) + _shfl_xor_sync_509; + float send_17 = ((b4_1694 == 0) ? dbp_1522[7] : dbp_1522[3]); + float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, send_17, 16); + v4_1697[3] = ((b4_1694 == 0) ? dbp_1522[3] : dbp_1522[7]) + _shfl_xor_sync_510; float v2_1702[2]; - float send_16 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); - float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_16, 8); - v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_507; - float send_17 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); - float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_508; + float send_18 = ((b3_1695 == 0) ? v4_1697[2] : v4_1697[0]); + float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_1702[0] = ((b3_1695 == 0) ? v4_1697[0] : v4_1697[2]) + _shfl_xor_sync_511; + float send_19 = ((b3_1695 == 0) ? v4_1697[3] : v4_1697[1]); + float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, send_19, 8); + v2_1702[1] = ((b3_1695 == 0) ? v4_1697[1] : v4_1697[3]) + _shfl_xor_sync_512; float send_1705 = ((b2_1696 == 0) ? v2_1702[1] : v2_1702[0]); - float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); - float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_509; - float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); - v1_1706 = v1_1706 + _shfl_xor_sync_510; - float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); - v1_1706 = v1_1706 + _shfl_xor_sync_511; + float _shfl_xor_sync_513 = __shfl_xor_sync(0xFFFFFFFF, send_1705, 4); + float v1_1706 = ((b2_1696 == 0) ? v2_1702[0] : v2_1702[1]) + _shfl_xor_sync_513; + float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 2); + v1_1706 = v1_1706 + _shfl_xor_sync_514; + float _shfl_xor_sync_515 = __shfl_xor_sync(0xFFFFFFFF, v1_1706, 1); + v1_1706 = v1_1706 + _shfl_xor_sync_515; tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_512 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_512), "f"(tot_1693) : "memory"); + uint32_t _addr_516 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_516), "f"(tot_1693) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5688,132 +5353,132 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_513 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_517 = make_float2(gacc[4], gacc[4]); float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_513), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_516; - asm("abs.f32 %0, %1;" : "=f"(_abs_516) : "f"(rp_z_11.x)); - float _neg_515; - asm("neg.f32 %0, %1;" : "=f"(_neg_515) : "f"(_abs_516)); - float _exp_514 = __nv_expf(_neg_515); - float _abs_519; - asm("abs.f32 %0, %1;" : "=f"(_abs_519) : "f"(rp_z_11.y)); - float _neg_518; - asm("neg.f32 %0, %1;" : "=f"(_neg_518) : "f"(_abs_519)); - float _exp_517 = __nv_expf(_neg_518); - float2 rp_e_11 = make_float2(_exp_514, _exp_517); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_517), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_520; + asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_11.x)); + float _neg_519; + asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); + float _exp_518 = __nv_expf(_neg_519); + float _abs_523; + asm("abs.f32 %0, %1;" : "=f"(_abs_523) : "f"(rp_z_11.y)); + float _neg_522; + asm("neg.f32 %0, %1;" : "=f"(_neg_522) : "f"(_abs_523)); + float _exp_521 = __nv_expf(_neg_522); + float2 rp_e_11 = make_float2(_exp_518, _exp_521); float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_520 = approx_rcp(rp_den_11.x); - float _rcp_521 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_522 = make_float2(_rcp_520, _rcp_521); + float _rcp_524 = approx_rcp(rp_den_11.x); + float _rcp_525 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_526 = make_float2(_rcp_524, _rcp_525); float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_522)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_526)); float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_523 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_527 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_527)); u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_524 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_528 = make_float2(gacc[4], gacc[4]); float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_524), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_527; - asm("abs.f32 %0, %1;" : "=f"(_abs_527) : "f"(rp_z_12.x)); - float _neg_526; - asm("neg.f32 %0, %1;" : "=f"(_neg_526) : "f"(_abs_527)); - float _exp_525 = __nv_expf(_neg_526); - float _abs_530; - asm("abs.f32 %0, %1;" : "=f"(_abs_530) : "f"(rp_z_12.y)); - float _neg_529; - asm("neg.f32 %0, %1;" : "=f"(_neg_529) : "f"(_abs_530)); - float _exp_528 = __nv_expf(_neg_529); - float2 rp_e_12 = make_float2(_exp_525, _exp_528); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_528), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_531; + asm("abs.f32 %0, %1;" : "=f"(_abs_531) : "f"(rp_z_12.x)); + float _neg_530; + asm("neg.f32 %0, %1;" : "=f"(_neg_530) : "f"(_abs_531)); + float _exp_529 = __nv_expf(_neg_530); + float _abs_534; + asm("abs.f32 %0, %1;" : "=f"(_abs_534) : "f"(rp_z_12.y)); + float _neg_533; + asm("neg.f32 %0, %1;" : "=f"(_neg_533) : "f"(_abs_534)); + float _exp_532 = __nv_expf(_neg_533); + float2 rp_e_12 = make_float2(_exp_529, _exp_532); float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_531 = approx_rcp(rp_den_12.x); - float _rcp_532 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_533 = make_float2(_rcp_531, _rcp_532); + float _rcp_535 = approx_rcp(rp_den_12.x); + float _rcp_536 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_537 = make_float2(_rcp_535, _rcp_536); float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_533)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_537)); float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_534 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_538 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_534)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_538)); u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_535 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_539 = make_float2(gacc[4], gacc[4]); float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_535), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_538; - asm("abs.f32 %0, %1;" : "=f"(_abs_538) : "f"(rp_z_13.x)); - float _neg_537; - asm("neg.f32 %0, %1;" : "=f"(_neg_537) : "f"(_abs_538)); - float _exp_536 = __nv_expf(_neg_537); - float _abs_541; - asm("abs.f32 %0, %1;" : "=f"(_abs_541) : "f"(rp_z_13.y)); - float _neg_540; - asm("neg.f32 %0, %1;" : "=f"(_neg_540) : "f"(_abs_541)); - float _exp_539 = __nv_expf(_neg_540); - float2 rp_e_13 = make_float2(_exp_536, _exp_539); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_539), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_542; + asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(rp_z_13.x)); + float _neg_541; + asm("neg.f32 %0, %1;" : "=f"(_neg_541) : "f"(_abs_542)); + float _exp_540 = __nv_expf(_neg_541); + float _abs_545; + asm("abs.f32 %0, %1;" : "=f"(_abs_545) : "f"(rp_z_13.y)); + float _neg_544; + asm("neg.f32 %0, %1;" : "=f"(_neg_544) : "f"(_abs_545)); + float _exp_543 = __nv_expf(_neg_544); + float2 rp_e_13 = make_float2(_exp_540, _exp_543); float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_542 = approx_rcp(rp_den_13.x); - float _rcp_543 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_544 = make_float2(_rcp_542, _rcp_543); + float _rcp_546 = approx_rcp(rp_den_13.x); + float _rcp_547 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_548 = make_float2(_rcp_546, _rcp_547); float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_544)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_548)); float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_545 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_549 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_545)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_549)); u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_546 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_550 = make_float2(gacc[4], gacc[4]); float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_546), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_549; - asm("abs.f32 %0, %1;" : "=f"(_abs_549) : "f"(rp_z_14.x)); - float _neg_548; - asm("neg.f32 %0, %1;" : "=f"(_neg_548) : "f"(_abs_549)); - float _exp_547 = __nv_expf(_neg_548); - float _abs_552; - asm("abs.f32 %0, %1;" : "=f"(_abs_552) : "f"(rp_z_14.y)); - float _neg_551; - asm("neg.f32 %0, %1;" : "=f"(_neg_551) : "f"(_abs_552)); - float _exp_550 = __nv_expf(_neg_551); - float2 rp_e_14 = make_float2(_exp_547, _exp_550); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_550), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_553; + asm("abs.f32 %0, %1;" : "=f"(_abs_553) : "f"(rp_z_14.x)); + float _neg_552; + asm("neg.f32 %0, %1;" : "=f"(_neg_552) : "f"(_abs_553)); + float _exp_551 = __nv_expf(_neg_552); + float _abs_556; + asm("abs.f32 %0, %1;" : "=f"(_abs_556) : "f"(rp_z_14.y)); + float _neg_555; + asm("neg.f32 %0, %1;" : "=f"(_neg_555) : "f"(_abs_556)); + float _exp_554 = __nv_expf(_neg_555); + float2 rp_e_14 = make_float2(_exp_551, _exp_554); float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_553 = approx_rcp(rp_den_14.x); - float _rcp_554 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_555 = make_float2(_rcp_553, _rcp_554); + float _rcp_557 = approx_rcp(rp_den_14.x); + float _rcp_558 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_559 = make_float2(_rcp_557, _rcp_558); float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_555)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_559)); float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_556 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_560 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_556)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_560)); u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -5828,54 +5493,54 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_557; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_557) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_557; + uint32_t _prmt_561; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_561; int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_558 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_558)); - *(reinterpret_cast(_gst_558) + (0)) = pq_12[0]; + auto _gst_562 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_562)); + *(reinterpret_cast(_gst_562) + (0)) = pq_12[0]; } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_559; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_559) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_559; + uint32_t _prmt_563; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_563; int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_560 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_560)); - *(reinterpret_cast(_gst_560) + (0)) = pq_13[0]; + auto _gst_564 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_564)); + *(reinterpret_cast(_gst_564) + (0)) = pq_13[0]; } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_561; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_561; + uint32_t _prmt_565; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_565) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_565; int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_562 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_562)); - *(reinterpret_cast(_gst_562) + (0)) = pq_14[0]; + auto _gst_566 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_566)); + *(reinterpret_cast(_gst_566) + (0)) = pq_14[0]; } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_563; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_563; + uint32_t _prmt_567; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_567) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_567; int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_564 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_564)); - *(reinterpret_cast(_gst_564) + (0)) = pq_15[0]; + auto _gst_568 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_568)); + *(reinterpret_cast(_gst_568) + (0)) = pq_15[0]; } } int t_a_24 = 8 * th; @@ -5940,47 +5605,47 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_565; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_565) : "f"(s_a_2), "f"(nm_t_5)); + float _sub_569; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_569) : "f"(s_a_2), "f"(nm_t_5)); float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_565), "f"(pa_2)); - float _sub_566; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_566) : "f"(pa_2), "f"(nm_t_5)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_569), "f"(pa_2)); + float _sub_570; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_570) : "f"(pa_2), "f"(nm_t_5)); float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_566), "f"(s_a_2)); - float _abs_568; - asm("abs.f32 %0, %1;" : "=f"(_abs_568) : "f"(s_a_2)); - float _abs_569; - asm("abs.f32 %0, %1;" : "=f"(_abs_569) : "f"(pa_2)); - uint32_t _setp_567_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_567_bits) : "f"(_abs_568), "f"(_abs_569)); - bool _setp_567 = _setp_567_bits != 0; - float nm_err_5 = ((_setp_567) ? nm_es_5 : nm_ex_5); - float _add_570; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_570; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_570), "f"(s_a_2)); + float _abs_572; + asm("abs.f32 %0, %1;" : "=f"(_abs_572) : "f"(s_a_2)); + float _abs_573; + asm("abs.f32 %0, %1;" : "=f"(_abs_573) : "f"(pa_2)); + uint32_t _setp_571_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_571_bits) : "f"(_abs_572), "f"(_abs_573)); + bool _setp_571 = _setp_571_bits != 0; + float nm_err_5 = ((_setp_571) ? nm_es_5 : nm_ex_5); + float _add_574; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_574; s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_571; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(s_b_2), "f"(nm_t_6)); + float _sub_575; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_575) : "f"(s_b_2), "f"(nm_t_6)); float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_571), "f"(pb_2)); - float _sub_572; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_572) : "f"(pb_2), "f"(nm_t_6)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_575), "f"(pb_2)); + float _sub_576; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_576) : "f"(pb_2), "f"(nm_t_6)); float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_572), "f"(s_b_2)); - float _abs_574; - asm("abs.f32 %0, %1;" : "=f"(_abs_574) : "f"(s_b_2)); - float _abs_575; - asm("abs.f32 %0, %1;" : "=f"(_abs_575) : "f"(pb_2)); - uint32_t _setp_573_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_573_bits) : "f"(_abs_574), "f"(_abs_575)); - bool _setp_573 = _setp_573_bits != 0; - float nm_err_6 = ((_setp_573) ? nm_es_6 : nm_ex_6); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_576; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_576), "f"(s_b_2)); + float _abs_578; + asm("abs.f32 %0, %1;" : "=f"(_abs_578) : "f"(s_b_2)); + float _abs_579; + asm("abs.f32 %0, %1;" : "=f"(_abs_579) : "f"(pb_2)); + uint32_t _setp_577_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_577_bits) : "f"(_abs_578), "f"(_abs_579)); + bool _setp_577 = _setp_577_bits != 0; + float nm_err_6 = ((_setp_577) ? nm_es_6 : nm_ex_6); + float _add_580; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_580; s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -6019,60 +5684,60 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_578; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_578) : "f"(ts_s), "f"(ts_bb)); - float _sub_577; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_577) : "f"(s_a_3), "f"(_sub_578)); - float _sub_579; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_579) : "f"(other[0]), "f"(ts_bb)); + float _sub_582; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s), "f"(ts_bb)); + float _sub_581; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(s_a_3), "f"(_sub_582)); + float _sub_583; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(other[0]), "f"(ts_bb)); float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_577), "f"(_sub_579)); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_a_3), "f"(other[1])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_581), "f"(_sub_583)); + float _add_584; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(c_a_3), "f"(other[1])); float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_580), "f"(ts_e)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_584), "f"(ts_e)); float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_582; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_581; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(ts_s), "f"(_sub_582)); - float _sub_583; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(pm_c), "f"(ts_bb_0)); + float _sub_586; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_585; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s), "f"(_sub_586)); + float _sub_587; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_587) : "f"(pm_c), "f"(ts_bb_0)); float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_581), "f"(_sub_583)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_585), "f"(_sub_587)); s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_585; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_584; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_584) : "f"(s_b_3), "f"(_sub_585)); - float _sub_586; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(other[2]), "f"(ts_bb_1)); + float _sub_589; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_588; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(s_b_3), "f"(_sub_589)); + float _sub_590; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(other[2]), "f"(ts_bb_1)); float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_584), "f"(_sub_586)); - float _add_587; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_587) : "f"(c_b_3), "f"(other[3])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_588), "f"(_sub_590)); + float _add_591; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_591) : "f"(c_b_3), "f"(other[3])); float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_587), "f"(ts_e_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_591), "f"(ts_e_1)); float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_589; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_588; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(ts_s_1), "f"(_sub_589)); - float _sub_590; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(pm_c_0), "f"(ts_bb_2)); + float _sub_593; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_593) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_592; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_592) : "f"(ts_s_1), "f"(_sub_593)); + float _sub_594; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_594) : "f"(pm_c_0), "f"(ts_bb_2)); float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_588), "f"(_sub_590)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_592), "f"(_sub_594)); s_b_3 = ts_s_2; c_b_3 = ts_e_2; gacc[0] = s_a_3; @@ -6093,7 +5758,7 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_1706; pool_f32[28160 + qd * 64 + 16 + 8 * th + e_star] = v1_1484; pool_f32[28160 + qd * 64 + 32 + 8 * th + e_star] = v1; - pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_1241; + pool_f32[28160 + qd * 64 + 48 + 8 * th + e_star] = v1_0; } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync 1, 256;" ::: "memory"); @@ -6183,41 +5848,41 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + for (int r_305 = 0; r_305 < 16; r_305++) { + int t_16 = r_305; + if (t_16 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6289,41 +5954,41 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_24 = 16 + r_348; - if (t_24 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + for (int r_306 = 0; r_306 < 16; r_306++) { + int t_17 = 16 + r_306; + if (t_17 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6439,41 +6104,41 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_25 = 32 + r_349; - if (t_25 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + for (int r_307 = 0; r_307 < 16; r_307++) { + int t_18 = 32 + r_307; + if (t_18 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6621,41 +6286,41 @@ kernel_cake_kda_train_ea14e16d1fc71364ba03(const __grid_constant__ CUtensorMap d z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_26 = 48 + r_350; - if (t_26 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + for (int r_308 = 0; r_308 < 16; r_308++) { + int t_19 = 48 + r_308; + if (t_19 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); From 1550268f6cccd429706b5ead0e927fc1ea896747 Mon Sep 17 00:00:00 2001 From: Yingyi Huang Date: Thu, 5 Nov 2026 11:51:02 -0800 Subject: [PATCH 3/5] cake_kda_train: re-render the S1 and R1c GLM intra programs (diagonal TF32 rounding, rolled drains) The parent commit copies the deterministic GLM intra programs after the diagonal TF32 rounding fix. This commit renders the package's own GLM intra programs from the matching generator sources, in place (same program names, new sources; the registry is unchanged): record program arch glm_intra_v21 d5fd3f7234a4266e09d2 sm_100a glm_intra_v22 f4ea302627f05d805437 sm_103a glm_intra_v23 8ce96eeb5b7517229400 sm_107a glm_intra_v20c 87b73f813c841159bf43 sm_100a, sm_103a glm_intra_v20c_r8 31fa936f5468b5749a33 sm_100a The S1 programs (v21 / v22 / v23) take the diagonal rounding of their deterministic twins; the R1c programs (v20c / v20c_r8) take it with their drains rolled. build.json carries the new source digests, identity closures and generator revision. The copies, the package's other programs and the KCP package are unchanged. A second render changes nothing, and the migration of the copies is still a fixed point. tests/experimental/test_cake_kda*.py on CPU: 2000 passed, 913 skipped (parent: the same). --- .../cake/cake_kda_train/build.json | 28 +- ...e_kda_train_31fa936f5468b5749a33_kernel.cu | 4003 +++++----- ...e_kda_train_87b73f813c841159bf43_kernel.cu | 6615 +++++++++++------ ...e_kda_train_8ce96eeb5b7517229400_kernel.cu | 1321 ++-- ...e_kda_train_d5fd3f7234a4266e09d2_kernel.cu | 1301 ++-- ...e_kda_train_f4ea302627f05d805437_kernel.cu | 1577 ++-- 6 files changed, 8803 insertions(+), 6042 deletions(-) diff --git a/flashinfer/experimental/cake/cake_kda_train/build.json b/flashinfer/experimental/cake/cake_kda_train/build.json index 912af147f76..5e2215eb81d 100644 --- a/flashinfer/experimental/cake/cake_kda_train/build.json +++ b/flashinfer/experimental/cake/cake_kda_train/build.json @@ -55,9 +55,9 @@ } }, "generator": { - "cake_revision": "0fb6c5f210b03ef4b08f171e574851ed6948c802", + "cake_revision": "b20c4b20c7eaf5ca5610279585cc850541151a70", "cuda": "13.4.92", - "rendered_at": "2026-11-03T20:20:58Z" + "rendered_at": "2026-11-05T11:26:08Z" }, "libraries": [ { @@ -1280,7 +1280,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "99462557e95324559ff8153af5f9bbd80b58e6e5671322fa6a2e9eb9df3e9529", + "closure_sha256": "2da34ebc77cbd5f81b4ddf2355ee8c5b7b45604cc45c65be7eab1f30661c2d8f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1292,7 +1292,7 @@ "path": "aot/cake_kda_train_31fa936f5468b5749a33_sm_100a.so", "sha256": { "csrc/cake_kda_train_31fa936f5468b5749a33_binding.cu": "acd15fe5c7bb7d68beae413a56148318728653988d6f38bdd6bed5a0ef5a1770", - "csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu": "b69b68c2545e612d5528dafeddea86b987d60cdda86b552d854ec2309719820a" + "csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu": "56730d7407f76b2987a440d706ad853711f6d5b9951f6ede099ccc174f0acc1f" }, "sources": [ "csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu", @@ -2372,7 +2372,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "997544e8154ec636d2523d6b93e8621d880a66b4fe38fd5e42dda40162afc702", + "closure_sha256": "29157dbfd3026aabb88e5598581e3286c2b09a2d35645d5083853d7966ea942b", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -2384,7 +2384,7 @@ "path": "aot/cake_kda_train_87b73f813c841159bf43_sm_100a.so", "sha256": { "csrc/cake_kda_train_87b73f813c841159bf43_binding.cu": "c0da28f7fa1049348c725e78a73f3b6dcdd10868184119a7f079f092cf315179", - "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu": "7ba3a424d66ed5ebbcc10f87d39dd83914a9ed6dd3f36dcbf4f72d5e1910f4fe" + "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu": "e22ac309ae36b47824d406eef60729bcc5abd6dd9ccd2db87904005664681133" }, "sources": [ "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu", @@ -2393,7 +2393,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "ea11b0bd27384f6857d205221258ec2947c158809aa114cb5313463ee79b99b3", + "closure_sha256": "be788de94c04b04343a128a62c63e97c0000737e0226cefb40e53cea054027bc", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -2405,7 +2405,7 @@ "path": "aot/cake_kda_train_87b73f813c841159bf43_sm_103a.so", "sha256": { "csrc/cake_kda_train_87b73f813c841159bf43_binding.cu": "c0da28f7fa1049348c725e78a73f3b6dcdd10868184119a7f079f092cf315179", - "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu": "7ba3a424d66ed5ebbcc10f87d39dd83914a9ed6dd3f36dcbf4f72d5e1910f4fe" + "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu": "e22ac309ae36b47824d406eef60729bcc5abd6dd9ccd2db87904005664681133" }, "sources": [ "csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu", @@ -2540,7 +2540,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "c9c2d8bd31381f2f1bd5546e1ed0e918382320dfa000480bfe7a2c79a3ecc604", + "closure_sha256": "1c7d14244747d80799cd40a99adec88417ae2a52fed7a95b0de30ea5c3ba7e1b", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -2552,7 +2552,7 @@ "path": "aot/cake_kda_train_8ce96eeb5b7517229400_sm_107a.so", "sha256": { "csrc/cake_kda_train_8ce96eeb5b7517229400_binding.cu": "3d3dbf0da8bfa77b603066798ee00288f4997c1d72cec5ff09083acc9bd546ec", - "csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu": "2257d9d1d6412d8c601b15c44e9a2493e4e0e64810e60d2803004ba712a2e776" + "csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu": "58e18a7c130094c763b82c97b6e9a7c20153afe9265e3b9aef41854ec336c4b6" }, "sources": [ "csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu", @@ -3842,7 +3842,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "a145e2981e4b53d084f181e059954472d86c3f7c9472dc592ec3af444acebb4e", + "closure_sha256": "66ed4fe1e61052a772c3cb4dea04cf1f01a79c2b3d8255c2f9e29166334ce700", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -3854,7 +3854,7 @@ "path": "aot/cake_kda_train_d5fd3f7234a4266e09d2_sm_100a.so", "sha256": { "csrc/cake_kda_train_d5fd3f7234a4266e09d2_binding.cu": "d64b14a931f8f57852c590ce0e8aa3c671dfd6b8ab718c53a4a6c98873dd8fb3", - "csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu": "2c6f0812dbd5db0216cb32155250fe6214e2273455c6254259b4b1610228f608" + "csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu": "92a99290857d4395558ddacca22dbf567907b2052528cf071a55b1925ede755a" }, "sources": [ "csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu", @@ -4640,7 +4640,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "012b974c3ab344ff46c357afbc659b26eb5d7d529f403355cec7ed50f914933b", + "closure_sha256": "0258bf6cf18edf2be46cbf176a04c9a773eac1bf6791b21e9089b0c05e65b46a", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -4652,7 +4652,7 @@ "path": "aot/cake_kda_train_f4ea302627f05d805437_sm_103a.so", "sha256": { "csrc/cake_kda_train_f4ea302627f05d805437_binding.cu": "92110639aeb17b136f85cf4fab2265c3a7efa549bd26b1aa142cd7f0cb28b0f0", - "csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu": "9b8a30bb07e1f13d83e7ddd87912691b8be4138fa51b0e7d99376b2ddebd912f" + "csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu": "9f0f17ebc89518adf4ed026985554ee0e458ef409907e1a2694b12b333225ed3" }, "sources": [ "csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu", diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu index f22e7eab9d7..a5fba7adf29 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_31fa936f5468b5749a33_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -199,6 +200,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2267,12 +2284,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_4); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_40); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_40, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_40 : __fmaf_rn(dl, _rcp_170, e2_40)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2296,12 +2319,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_5); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_41); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_41, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_41, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_41 : __fmaf_rn(dl_1, _rcp_174, e2_41)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2325,12 +2354,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_6); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_42); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_42, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_42, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_42 : __fmaf_rn(dl_3, _rcp_178, e2_42)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2354,12 +2389,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_7); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_43); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_43, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_43, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_43 : __fmaf_rn(dl_5, _rcp_182, e2_43)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2395,12 +2436,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2420,12 +2467,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2445,12 +2498,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2470,12 +2529,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2523,12 +2588,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_0 = sub_f32x2(_f2_64, _f2_8); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2552,12 +2623,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_1 = sub_f32x2(_f2_64, _f2_9); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2581,12 +2658,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_2 = sub_f32x2(_f2_64, _f2_10); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2610,12 +2693,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_3 = sub_f32x2(_f2_64, _f2_11); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2657,12 +2746,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_4 = sub_f32x2(_f2_70, _f2_12); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2686,12 +2781,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_5 = sub_f32x2(_f2_70, _f2_13); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2715,12 +2816,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_6 = sub_f32x2(_f2_70, _f2_14); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2744,12 +2851,18 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 dn2_7 = sub_f32x2(_f2_70, _f2_15); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2820,37 +2933,37 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_60.x); float e1_12 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_61 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_61.x); float e1_13 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_62 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_62.x); float e1_14 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_63 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_63.x); float e1_15 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2862,29 +2975,29 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2892,29 +3005,29 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2940,402 +3053,133 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int sb8 = 32 + 8 * th; int gb8 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8 * 128); int kb8 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8 * 128); - float dbp[8]; - float qo[8]; - tmem_ld_x8(&qo[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float ko[8]; - tmem_ld_x8(&ko[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float qdg[8]; - tmem_ld_x8(&qdg[0], taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)); - float kdg[8]; - tmem_ld_x8(&kdg[0], taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th)); - float dkt_0[8]; - tmem_ld_x8(&dkt_0[0], taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th)); - float btv[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[0])), "=r"(*reinterpret_cast(&btv[(0) + 1])), "=r"(*reinterpret_cast(&btv[(0) + 2])), "=r"(*reinterpret_cast(&btv[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[4])), "=r"(*reinterpret_cast(&btv[(4) + 1])), "=r"(*reinterpret_cast(&btv[(4) + 2])), "=r"(*reinterpret_cast(&btv[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int c_bp = 144 + 8 * th; float s_run[1]; s_run[0] = 0.0f; - int t_0 = 32 + 8 * th + 7; - float r_248[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_248[0]; - unsigned int r_249[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_249[0] << 16); - unsigned int r_250[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_250[0] << 16); - float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew + 896 + d_lane]; - float dkf_0 = pool_f32[ew + 1920 + d_lane]; - float dgf_0 = pool_f32[ew + 2944 + d_lane]; - float bt_t_0 = btv[7]; - float e_off_0 = approx_exp2(g_t_0 - gn); - float e_dg_0 = approx_exp2(g_t_0 - gc_0); - float dq2_0 = __fmaf_rn(qo[7], e_off_0, qdg[7] * e_dg_0); - float dk2_0 = __fmaf_rn(ko[7], e_off_0, kdg[7] * e_dg_0); - dbp[7] = dk2_0 * k_t_0; - float dk2b_0 = dk2_0 * bt_t_0; - float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + dkt_0[7]; - float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, (dk2b_0 - dkt_0[7]) * k_t_0) + dgf_0; - s_run[0] = s_run[0] + dg_o_0; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 7)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_0 = t_0 * qrs; - if (t_0 < clen) { - { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; - } - { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; - } - } - int t_1 = 32 + 8 * th + 6; - float r_251[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_251[0]; - unsigned int r_252[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((kb8 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_252[0] << 16); - unsigned int r_253[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_253[0] << 16); - float q_t_1_1 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew + 768 + d_lane]; - float dkf_1 = pool_f32[ew + 1792 + d_lane]; - float dgf_1 = pool_f32[ew + 2816 + d_lane]; - float bt_t_1 = btv[6]; - float e_off_1 = approx_exp2(g_t_1 - gn); - float e_dg_1 = approx_exp2(g_t_1 - gc_0); - float dq2_1 = __fmaf_rn(qo[6], e_off_1, qdg[6] * e_dg_1); - float dk2_1 = __fmaf_rn(ko[6], e_off_1, kdg[6] * e_dg_1); - dbp[6] = dk2_1 * k_t_1; - float dk2b_1 = dk2_1 * bt_t_1; - float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + dkt_0[6]; - float dg_o_1 = __fmaf_rn(q_t_1_1, dq2_1, (dk2b_1 - dkt_0[6]) * k_t_1) + dgf_1; - s_run[0] = s_run[0] + dg_o_1; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 6)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_1 = t_1 * qrs; - if (t_1 < clen) { - { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; - } - { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; - } - } - int t_2 = 32 + 8 * th + 5; - float r_254[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_254[0]; - unsigned int r_255[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((kb8 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_255[0] << 16); - unsigned int r_256[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_256[0] << 16); - float q_t_1_2 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew + 640 + d_lane]; - float dkf_2 = pool_f32[ew + 1664 + d_lane]; - float dgf_2 = pool_f32[ew + 2688 + d_lane]; - float bt_t_2 = btv[5]; - float e_off_2 = approx_exp2(g_t_2 - gn); - float e_dg_2 = approx_exp2(g_t_2 - gc_0); - float dq2_2 = __fmaf_rn(qo[5], e_off_2, qdg[5] * e_dg_2); - float dk2_2 = __fmaf_rn(ko[5], e_off_2, kdg[5] * e_dg_2); - dbp[5] = dk2_2 * k_t_2; - float dk2b_2 = dk2_2 * bt_t_2; - float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + dkt_0[5]; - float dg_o_2 = __fmaf_rn(q_t_1_2, dq2_2, (dk2b_2 - dkt_0[5]) * k_t_2) + dgf_2; - s_run[0] = s_run[0] + dg_o_2; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 5)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_2 = t_2 * qrs; - if (t_2 < clen) { - { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; - } - { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; - } - } - int t_3 = 32 + 8 * th + 4; - float r_257[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_257[0]; - unsigned int r_258[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((kb8 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_258[0] << 16); - unsigned int r_259[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_259[0] << 16); - float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew + 512 + d_lane]; - float dkf_3 = pool_f32[ew + 1536 + d_lane]; - float dgf_3 = pool_f32[ew + 2560 + d_lane]; - float bt_t_3 = btv[4]; - float e_off_3 = approx_exp2(g_t_3 - gn); - float e_dg_3 = approx_exp2(g_t_3 - gc_0); - float dq2_3 = __fmaf_rn(qo[4], e_off_3, qdg[4] * e_dg_3); - float dk2_3 = __fmaf_rn(ko[4], e_off_3, kdg[4] * e_dg_3); - dbp[4] = dk2_3 * k_t_3; - float dk2b_3 = dk2_3 * bt_t_3; - float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + dkt_0[4]; - float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, (dk2b_3 - dkt_0[4]) * k_t_3) + dgf_3; - s_run[0] = s_run[0] + dg_o_3; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 4)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_3 = t_3 * qrs; - if (t_3 < clen) { - { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; - } - { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; - } - } - int t_4 = 32 + 8 * th + 3; - float r_260[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_260[0]; - unsigned int r_261[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((kb8 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_261[0] << 16); - unsigned int r_262[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_262[0] << 16); - float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew + 384 + d_lane]; - float dkf_4 = pool_f32[ew + 1408 + d_lane]; - float dgf_4 = pool_f32[ew + 2432 + d_lane]; - float bt_t_4 = btv[3]; - float e_off_4 = approx_exp2(g_t_4 - gn); - float e_dg_4 = approx_exp2(g_t_4 - gc_0); - float dq2_4 = __fmaf_rn(qo[3], e_off_4, qdg[3] * e_dg_4); - float dk2_4 = __fmaf_rn(ko[3], e_off_4, kdg[3] * e_dg_4); - dbp[3] = dk2_4 * k_t_4; - float dk2b_4 = dk2_4 * bt_t_4; - float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + dkt_0[3]; - float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, (dk2b_4 - dkt_0[3]) * k_t_4) + dgf_4; - s_run[0] = s_run[0] + dg_o_4; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 3)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_4 = t_4 * qrs; - if (t_4 < clen) { - { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; - } - { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; - } - } - int t_5 = 32 + 8 * th + 2; - float r_263[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_263[0]; - unsigned int r_264[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((kb8 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_264[0] << 16); - unsigned int r_265[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_265[0] << 16); - float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew + 256 + d_lane]; - float dkf_5 = pool_f32[ew + 1280 + d_lane]; - float dgf_5 = pool_f32[ew + 2304 + d_lane]; - float bt_t_5 = btv[2]; - float e_off_5 = approx_exp2(g_t_5 - gn); - float e_dg_5 = approx_exp2(g_t_5 - gc_0); - float dq2_5 = __fmaf_rn(qo[2], e_off_5, qdg[2] * e_dg_5); - float dk2_5 = __fmaf_rn(ko[2], e_off_5, kdg[2] * e_dg_5); - dbp[2] = dk2_5 * k_t_5; - float dk2b_5 = dk2_5 * bt_t_5; - float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + dkt_0[2]; - float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, (dk2b_5 - dkt_0[2]) * k_t_5) + dgf_5; - s_run[0] = s_run[0] + dg_o_5; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 2)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_5 = t_5 * qrs; - if (t_5 < clen) { - { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; - } - { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; - } - } - int t_6 = 32 + 8 * th + 1; - float r_266[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_266[0]; - unsigned int r_267[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_267[0] << 16); - unsigned int r_268[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_268[0] << 16); - float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew + 128 + d_lane]; - float dkf_6 = pool_f32[ew + 1152 + d_lane]; - float dgf_6 = pool_f32[ew + 2176 + d_lane]; - float bt_t_6 = btv[1]; - float e_off_6 = approx_exp2(g_t_6 - gn); - float e_dg_6 = approx_exp2(g_t_6 - gc_0); - float dq2_6 = __fmaf_rn(qo[1], e_off_6, qdg[1] * e_dg_6); - float dk2_6 = __fmaf_rn(ko[1], e_off_6, kdg[1] * e_dg_6); - dbp[1] = dk2_6 * k_t_6; - float dk2b_6 = dk2_6 * bt_t_6; - float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + dkt_0[1]; - float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, (dk2b_6 - dkt_0[1]) * k_t_6) + dgf_6; - s_run[0] = s_run[0] + dg_o_6; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 1)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_6 = t_6 * qrs; - if (t_6 < clen) { - { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; - } - { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; - } - } - int t_7 = 32 + 8 * th; - float r_269[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_269[0]; - unsigned int r_270[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((kb8 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_270[0] << 16); - unsigned int r_271[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_271[0] << 16); - float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew + d_lane]; - float dkf_7 = pool_f32[ew + 1024 + d_lane]; - float dgf_7 = pool_f32[ew + 2048 + d_lane]; - float bt_t_7 = btv[0]; - float e_off_7 = approx_exp2(g_t_7 - gn); - float e_dg_7 = approx_exp2(g_t_7 - gc_0); - float dq2_7 = __fmaf_rn(qo[0], e_off_7, qdg[0] * e_dg_7); - float dk2_7 = __fmaf_rn(ko[0], e_off_7, kdg[0] * e_dg_7); - dbp[0] = dk2_7 * k_t_7; - float dk2b_7 = dk2_7 * bt_t_7; - float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + dkt_0[0]; - float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, (dk2b_7 - dkt_0[0]) * k_t_7) + dgf_7; - s_run[0] = s_run[0] + dg_o_7; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)), "f"(s_run[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_7 = t_7 * qrs; - if (t_7 < clen) { - { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; - } - { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j = 7 - k; + float qo[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qo[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + j))); + float ko[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(ko[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th + j))); + float qdg[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qdg[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + j))); + float kdg[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(kdg[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th + j))); + float dkt_0[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(dkt_0[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th + j))); + float btv[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btv[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th + j)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8 + j; + float r_248[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_248[0]; + unsigned int r_249[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_249[0] << 16); + unsigned int r_250[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_250[0] << 16); + float q_t_1 = ((t >= clen) ? 0.0f : q_t); + float bt_t = btv[0]; + float e_off = approx_exp2(g_t - gn); + float e_dg = approx_exp2(g_t - gc_0); + float dq2 = __fmaf_rn(qo[0], e_off, qdg[0] * e_dg); + float dk2 = __fmaf_rn(ko[0], e_off, kdg[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float qe = q_t_1 * e_dg; + float kp = k_t * bt_t; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg) * kdg[0]); + float dqf = pool_f32[ew + 128 * j + d_lane]; + float dkf = pool_f32[ew + (1024 + 128 * j) + d_lane]; + float dgf = pool_f32[ew + (2048 + 128 * j) + d_lane]; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + dkt_0[0]; + float dg_o = __fmaf_rn(q_t_1, dq2, __fmaf_rn(dk2b - dkt_0[0], k_t, cr)) + dgf; + s_run[0] = s_run[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + j)), "f"(s_run[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff = t * qrs; + if (t < clen) { + { + auto _gst_248 = dq_base + qoff; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o; + } + { + auto _gst_249 = dk_base + qoff; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o; + } } } float tot = s_run[0]; + float dbp[8]; + tmem_ld_x8(&dbp[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); int b4 = lane_1 >> 4 & 1; int b3 = lane_1 >> 3 & 1; int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_250; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_251; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_252; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_253; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_254; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_255; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_256; + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_257; + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_258; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3346,9 +3190,9 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d if (lane_1 == 17) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(dt_bias + (hv * 128 + 32 * qd)))); } mbarrier_wait(epi_full_addr + (th) * 8, 1); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_272[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_0 = r_272[0]; + float r_251[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_0 = r_251[0]; float gc_1 = gn_0 - 21.640425f; int eb_0 = 98304 + th * -32768; int ew_0 = eb_0 >> 2; @@ -3356,402 +3200,133 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int sb8_0 = 48 + 8 * th; int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); - float dbp_0[8]; - float qo_0[8]; - tmem_ld_x8(&qo_0[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float ko_0[8]; - tmem_ld_x8(&ko_0[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float qdg_0[8]; - tmem_ld_x8(&qdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float kdg_0[8]; - tmem_ld_x8(&kdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float dkt_1[8]; - tmem_ld_x8(&dkt_1[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_0[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_0[0])), "=r"(*reinterpret_cast(&btv_0[(0) + 1])), "=r"(*reinterpret_cast(&btv_0[(0) + 2])), "=r"(*reinterpret_cast(&btv_0[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_0[4])), "=r"(*reinterpret_cast(&btv_0[(4) + 1])), "=r"(*reinterpret_cast(&btv_0[(4) + 2])), "=r"(*reinterpret_cast(&btv_0[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int c_bp_0 = 176 + 8 * th; float s_run_0[1]; s_run_0[0] = 0.0f; - int t_8 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_0 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_0 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_0 + 2944 + d_lane]; - float bt_t_8 = btv_0[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_0); - float e_dg_8 = approx_exp2(g_t_8 - gc_1); - float dq2_8 = __fmaf_rn(qo_0[7], e_off_8, qdg_0[7] * e_dg_8); - float dk2_8 = __fmaf_rn(ko_0[7], e_off_8, kdg_0[7] * e_dg_8); - dbp_0[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + dkt_1[7]; - float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, (dk2b_8 - dkt_1[7]) * k_t_8) + dgf_8; - s_run_0[0] = s_run_0[0] + dg_o_8; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_8 = t_8 * qrs; - if (t_8 < clen) { - { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; - } - { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; - } - } - int t_9 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_0 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_0 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_0 + 2816 + d_lane]; - float bt_t_9 = btv_0[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_0); - float e_dg_9 = approx_exp2(g_t_9 - gc_1); - float dq2_9 = __fmaf_rn(qo_0[6], e_off_9, qdg_0[6] * e_dg_9); - float dk2_9 = __fmaf_rn(ko_0[6], e_off_9, kdg_0[6] * e_dg_9); - dbp_0[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + dkt_1[6]; - float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, (dk2b_9 - dkt_1[6]) * k_t_9) + dgf_9; - s_run_0[0] = s_run_0[0] + dg_o_9; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_9 = t_9 * qrs; - if (t_9 < clen) { - { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; - } - { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; - } - } - int t_10 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_0 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_0 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_0 + 2688 + d_lane]; - float bt_t_10 = btv_0[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_0); - float e_dg_10 = approx_exp2(g_t_10 - gc_1); - float dq2_10 = __fmaf_rn(qo_0[5], e_off_10, qdg_0[5] * e_dg_10); - float dk2_10 = __fmaf_rn(ko_0[5], e_off_10, kdg_0[5] * e_dg_10); - dbp_0[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + dkt_1[5]; - float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, (dk2b_10 - dkt_1[5]) * k_t_10) + dgf_10; - s_run_0[0] = s_run_0[0] + dg_o_10; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_10 = t_10 * qrs; - if (t_10 < clen) { - { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; - } - { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; - } - } - int t_11 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_0 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_0 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_0 + 2560 + d_lane]; - float bt_t_11 = btv_0[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_0); - float e_dg_11 = approx_exp2(g_t_11 - gc_1); - float dq2_11 = __fmaf_rn(qo_0[4], e_off_11, qdg_0[4] * e_dg_11); - float dk2_11 = __fmaf_rn(ko_0[4], e_off_11, kdg_0[4] * e_dg_11); - dbp_0[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + dkt_1[4]; - float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, (dk2b_11 - dkt_1[4]) * k_t_11) + dgf_11; - s_run_0[0] = s_run_0[0] + dg_o_11; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_11 = t_11 * qrs; - if (t_11 < clen) { - { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; - } - { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; - } - } - int t_12 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_0 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_0 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_0 + 2432 + d_lane]; - float bt_t_12 = btv_0[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_0); - float e_dg_12 = approx_exp2(g_t_12 - gc_1); - float dq2_12 = __fmaf_rn(qo_0[3], e_off_12, qdg_0[3] * e_dg_12); - float dk2_12 = __fmaf_rn(ko_0[3], e_off_12, kdg_0[3] * e_dg_12); - dbp_0[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + dkt_1[3]; - float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, (dk2b_12 - dkt_1[3]) * k_t_12) + dgf_12; - s_run_0[0] = s_run_0[0] + dg_o_12; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_12 = t_12 * qrs; - if (t_12 < clen) { - { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; - } - { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; - } - } - int t_13 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_0 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_0 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_0 + 2304 + d_lane]; - float bt_t_13 = btv_0[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_0); - float e_dg_13 = approx_exp2(g_t_13 - gc_1); - float dq2_13 = __fmaf_rn(qo_0[2], e_off_13, qdg_0[2] * e_dg_13); - float dk2_13 = __fmaf_rn(ko_0[2], e_off_13, kdg_0[2] * e_dg_13); - dbp_0[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + dkt_1[2]; - float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, (dk2b_13 - dkt_1[2]) * k_t_13) + dgf_13; - s_run_0[0] = s_run_0[0] + dg_o_13; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_13 = t_13 * qrs; - if (t_13 < clen) { - { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; - } - { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; - } - } - int t_14 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_0 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_0 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_0 + 2176 + d_lane]; - float bt_t_14 = btv_0[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_0); - float e_dg_14 = approx_exp2(g_t_14 - gc_1); - float dq2_14 = __fmaf_rn(qo_0[1], e_off_14, qdg_0[1] * e_dg_14); - float dk2_14 = __fmaf_rn(ko_0[1], e_off_14, kdg_0[1] * e_dg_14); - dbp_0[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + dkt_1[1]; - float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, (dk2b_14 - dkt_1[1]) * k_t_14) + dgf_14; - s_run_0[0] = s_run_0[0] + dg_o_14; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_14 = t_14 * qrs; - if (t_14 < clen) { - { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; - } - { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; - } - } - int t_15 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_0 + d_lane]; - float dkf_15 = pool_f32[ew_0 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_0 + 2048 + d_lane]; - float bt_t_15 = btv_0[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_0); - float e_dg_15 = approx_exp2(g_t_15 - gc_1); - float dq2_15 = __fmaf_rn(qo_0[0], e_off_15, qdg_0[0] * e_dg_15); - float dk2_15 = __fmaf_rn(ko_0[0], e_off_15, kdg_0[0] * e_dg_15); - dbp_0[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + dkt_1[0]; - float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, (dk2b_15 - dkt_1[0]) * k_t_15) + dgf_15; - s_run_0[0] = s_run_0[0] + dg_o_15; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_15 = t_15 * qrs; - if (t_15 < clen) { - { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; - } - { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_1 = 7 - k_1; + float qo_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qo_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th + j_1))); + float ko_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(ko_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th + j_1))); + float qdg_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qdg_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + j_1))); + float kdg_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(kdg_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th + j_1))); + float dkt_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(dkt_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th + j_1))); + float btv_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btv_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_0 + j_1; + float r_252[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_252[0]; + unsigned int r_253[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_253[0] << 16); + unsigned int r_254[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_254[0] << 16); + float q_t_1_1 = ((t_1 >= clen) ? 0.0f : q_t_2); + float bt_t_1 = btv_1[0]; + float e_off_1 = approx_exp2(g_t_1 - gn_0); + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float dq2_1 = __fmaf_rn(qo_1[0], e_off_1, qdg_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(ko_1[0], e_off_1, kdg_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float qe_1 = q_t_1_1 * e_dg_1; + float kp_12 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_12; + float kbe_1 = (float)kb_1 * e_dg_1; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg_1[0], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_12 * e_dg_1) * kdg_1[0]); + float dqf_1 = pool_f32[ew_0 + 128 * j_1 + d_lane]; + float dkf_1 = pool_f32[ew_0 + (1024 + 128 * j_1) + d_lane]; + float dgf_1 = pool_f32[ew_0 + (2048 + 128 * j_1) + d_lane]; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + dkt_1[0]; + float dg_o_1 = __fmaf_rn(q_t_1_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_1[0], k_t_1, cr_1)) + dgf_1; + s_run_0[0] = s_run_0[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + j_1)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_1)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff_1 = t_1 * qrs; + if (t_1 < clen) { + { + auto _gst_259 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_1; + } + { + auto _gst_260 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_1; + } } } float tot_0 = s_run_0[0]; + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); int b4_0 = lane_1 >> 4 & 1; int b3_0 = lane_1 >> 3 & 1; int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_261; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_262; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_263; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_264; float v2_16[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_265; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_266; float send_12 = ((b2_0 == 0) ? v2_16[1] : v2_16[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_267; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_268; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_269; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); float gacc[8]; @@ -3803,8 +3378,8 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); + uint32_t _addr_270 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_270), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3836,132 +3411,132 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_267 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_271 = make_float2(gacc[4], gacc[4]); float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&rp_u)); - float _abs_270; - asm("abs.f32 %0, %1;" : "=f"(_abs_270) : "f"(rp_z.x)); - float _neg_269; - asm("neg.f32 %0, %1;" : "=f"(_neg_269) : "f"(_abs_270)); - float _exp_268 = __nv_expf(_neg_269); - float _abs_273; - asm("abs.f32 %0, %1;" : "=f"(_abs_273) : "f"(rp_z.y)); - float _neg_272; - asm("neg.f32 %0, %1;" : "=f"(_neg_272) : "f"(_abs_273)); - float _exp_271 = __nv_expf(_neg_272); - float2 rp_e = make_float2(_exp_268, _exp_271); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_271), "l"(*(const unsigned long long*)&rp_u)); + float _abs_274; + asm("abs.f32 %0, %1;" : "=f"(_abs_274) : "f"(rp_z.x)); + float _neg_273; + asm("neg.f32 %0, %1;" : "=f"(_neg_273) : "f"(_abs_274)); + float _exp_272 = __nv_expf(_neg_273); + float _abs_277; + asm("abs.f32 %0, %1;" : "=f"(_abs_277) : "f"(rp_z.y)); + float _neg_276; + asm("neg.f32 %0, %1;" : "=f"(_neg_276) : "f"(_abs_277)); + float _exp_275 = __nv_expf(_neg_276); + float2 rp_e = make_float2(_exp_272, _exp_275); float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_274 = approx_rcp(rp_den.x); - float _rcp_275 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_276 = make_float2(_rcp_274, _rcp_275); + float _rcp_278 = approx_rcp(rp_den.x); + float _rcp_279 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_280 = make_float2(_rcp_278, _rcp_279); float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_276)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_280)); float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_277 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_281 = make_float2(gacc[5], gacc[5]); float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_281)); u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_278 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_282 = make_float2(gacc[4], gacc[4]); float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_278), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_281; - asm("abs.f32 %0, %1;" : "=f"(_abs_281) : "f"(rp_z_0.x)); - float _neg_280; - asm("neg.f32 %0, %1;" : "=f"(_neg_280) : "f"(_abs_281)); - float _exp_279 = __nv_expf(_neg_280); - float _abs_284; - asm("abs.f32 %0, %1;" : "=f"(_abs_284) : "f"(rp_z_0.y)); - float _neg_283; - asm("neg.f32 %0, %1;" : "=f"(_neg_283) : "f"(_abs_284)); - float _exp_282 = __nv_expf(_neg_283); - float2 rp_e_0 = make_float2(_exp_279, _exp_282); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_282), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_285; + asm("abs.f32 %0, %1;" : "=f"(_abs_285) : "f"(rp_z_0.x)); + float _neg_284; + asm("neg.f32 %0, %1;" : "=f"(_neg_284) : "f"(_abs_285)); + float _exp_283 = __nv_expf(_neg_284); + float _abs_288; + asm("abs.f32 %0, %1;" : "=f"(_abs_288) : "f"(rp_z_0.y)); + float _neg_287; + asm("neg.f32 %0, %1;" : "=f"(_neg_287) : "f"(_abs_288)); + float _exp_286 = __nv_expf(_neg_287); + float2 rp_e_0 = make_float2(_exp_283, _exp_286); float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_285 = approx_rcp(rp_den_0.x); - float _rcp_286 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_287 = make_float2(_rcp_285, _rcp_286); + float _rcp_289 = approx_rcp(rp_den_0.x); + float _rcp_290 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_291 = make_float2(_rcp_289, _rcp_290); float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_287)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_291)); float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_288 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_292 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_288)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_292)); u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_289 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_293 = make_float2(gacc[4], gacc[4]); float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_289), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_292; - asm("abs.f32 %0, %1;" : "=f"(_abs_292) : "f"(rp_z_1.x)); - float _neg_291; - asm("neg.f32 %0, %1;" : "=f"(_neg_291) : "f"(_abs_292)); - float _exp_290 = __nv_expf(_neg_291); - float _abs_295; - asm("abs.f32 %0, %1;" : "=f"(_abs_295) : "f"(rp_z_1.y)); - float _neg_294; - asm("neg.f32 %0, %1;" : "=f"(_neg_294) : "f"(_abs_295)); - float _exp_293 = __nv_expf(_neg_294); - float2 rp_e_1 = make_float2(_exp_290, _exp_293); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_293), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_296; + asm("abs.f32 %0, %1;" : "=f"(_abs_296) : "f"(rp_z_1.x)); + float _neg_295; + asm("neg.f32 %0, %1;" : "=f"(_neg_295) : "f"(_abs_296)); + float _exp_294 = __nv_expf(_neg_295); + float _abs_299; + asm("abs.f32 %0, %1;" : "=f"(_abs_299) : "f"(rp_z_1.y)); + float _neg_298; + asm("neg.f32 %0, %1;" : "=f"(_neg_298) : "f"(_abs_299)); + float _exp_297 = __nv_expf(_neg_298); + float2 rp_e_1 = make_float2(_exp_294, _exp_297); float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_296 = approx_rcp(rp_den_1.x); - float _rcp_297 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_298 = make_float2(_rcp_296, _rcp_297); + float _rcp_300 = approx_rcp(rp_den_1.x); + float _rcp_301 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_302 = make_float2(_rcp_300, _rcp_301); float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_298)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_302)); float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_299 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_303 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_299)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_303)); u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_300 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_304 = make_float2(gacc[4], gacc[4]); float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_303; - asm("abs.f32 %0, %1;" : "=f"(_abs_303) : "f"(rp_z_2.x)); - float _neg_302; - asm("neg.f32 %0, %1;" : "=f"(_neg_302) : "f"(_abs_303)); - float _exp_301 = __nv_expf(_neg_302); - float _abs_306; - asm("abs.f32 %0, %1;" : "=f"(_abs_306) : "f"(rp_z_2.y)); - float _neg_305; - asm("neg.f32 %0, %1;" : "=f"(_neg_305) : "f"(_abs_306)); - float _exp_304 = __nv_expf(_neg_305); - float2 rp_e_2 = make_float2(_exp_301, _exp_304); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_304), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_307; + asm("abs.f32 %0, %1;" : "=f"(_abs_307) : "f"(rp_z_2.x)); + float _neg_306; + asm("neg.f32 %0, %1;" : "=f"(_neg_306) : "f"(_abs_307)); + float _exp_305 = __nv_expf(_neg_306); + float _abs_310; + asm("abs.f32 %0, %1;" : "=f"(_abs_310) : "f"(rp_z_2.y)); + float _neg_309; + asm("neg.f32 %0, %1;" : "=f"(_neg_309) : "f"(_abs_310)); + float _exp_308 = __nv_expf(_neg_309); + float2 rp_e_2 = make_float2(_exp_305, _exp_308); float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_307 = approx_rcp(rp_den_2.x); - float _rcp_308 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_309 = make_float2(_rcp_307, _rcp_308); + float _rcp_311 = approx_rcp(rp_den_2.x); + float _rcp_312 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_313 = make_float2(_rcp_311, _rcp_312); float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_313)); float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_310 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_314 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_310)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_314)); u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -3976,54 +3551,54 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_311; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_311) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_311; + uint32_t _prmt_315; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_315; int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_312 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = pq_0[0]; + auto _gst_316 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = pq_0[0]; } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_313; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_313) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_313; + uint32_t _prmt_317; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_317; int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_314 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = pq_1[0]; + auto _gst_318 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = pq_1[0]; } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_315; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_315; + uint32_t _prmt_319; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_319) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_319; int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_316 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = pq_2[0]; + auto _gst_320 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = pq_2[0]; } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_317; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_317; + uint32_t _prmt_321; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_321) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_321; int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_318 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = pq_3[0]; + auto _gst_322 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast(_gst_322) + (0)) = pq_3[0]; } } int t_a_0 = 48 + 8 * th; @@ -4088,47 +3663,47 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_319; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_319) : "f"(s_a), "f"(nm_t)); + float _sub_323; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_323) : "f"(s_a), "f"(nm_t)); float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_319), "f"(pa)); - float _sub_320; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_320) : "f"(pa), "f"(nm_t)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_323), "f"(pa)); + float _sub_324; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_324) : "f"(pa), "f"(nm_t)); float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_320), "f"(s_a)); - float _abs_322; - asm("abs.f32 %0, %1;" : "=f"(_abs_322) : "f"(s_a)); - float _abs_323; - asm("abs.f32 %0, %1;" : "=f"(_abs_323) : "f"(pa)); - uint32_t _setp_321_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_321_bits) : "f"(_abs_322), "f"(_abs_323)); - bool _setp_321 = _setp_321_bits != 0; - float nm_err = ((_setp_321) ? nm_es : nm_ex); - float _add_324; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(c_a), "f"(nm_err)); - c_a = _add_324; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_324), "f"(s_a)); + float _abs_326; + asm("abs.f32 %0, %1;" : "=f"(_abs_326) : "f"(s_a)); + float _abs_327; + asm("abs.f32 %0, %1;" : "=f"(_abs_327) : "f"(pa)); + uint32_t _setp_325_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_325_bits) : "f"(_abs_326), "f"(_abs_327)); + bool _setp_325 = _setp_325_bits != 0; + float nm_err = ((_setp_325) ? nm_es : nm_ex); + float _add_328; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(c_a), "f"(nm_err)); + c_a = _add_328; s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_325; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_325) : "f"(s_b), "f"(nm_t_0)); + float _sub_329; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_329) : "f"(s_b), "f"(nm_t_0)); float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_325), "f"(pb)); - float _sub_326; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_326) : "f"(pb), "f"(nm_t_0)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_329), "f"(pb)); + float _sub_330; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_330) : "f"(pb), "f"(nm_t_0)); float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_326), "f"(s_b)); - float _abs_328; - asm("abs.f32 %0, %1;" : "=f"(_abs_328) : "f"(s_b)); - float _abs_329; - asm("abs.f32 %0, %1;" : "=f"(_abs_329) : "f"(pb)); - uint32_t _setp_327_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_327_bits) : "f"(_abs_328), "f"(_abs_329)); - bool _setp_327 = _setp_327_bits != 0; - float nm_err_0 = ((_setp_327) ? nm_es_0 : nm_ex_0); - float _add_330; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_330; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_330), "f"(s_b)); + float _abs_332; + asm("abs.f32 %0, %1;" : "=f"(_abs_332) : "f"(s_b)); + float _abs_333; + asm("abs.f32 %0, %1;" : "=f"(_abs_333) : "f"(pb)); + uint32_t _setp_331_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_331_bits) : "f"(_abs_332), "f"(_abs_333)); + bool _setp_331 = _setp_331_bits != 0; + float nm_err_0 = ((_setp_331) ? nm_es_0 : nm_ex_0); + float _add_334; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_334; s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4152,132 +3727,132 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_331 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_335 = make_float2(gacc[4], gacc[4]); float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_331), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_334; - asm("abs.f32 %0, %1;" : "=f"(_abs_334) : "f"(rp_z_3.x)); - float _neg_333; - asm("neg.f32 %0, %1;" : "=f"(_neg_333) : "f"(_abs_334)); - float _exp_332 = __nv_expf(_neg_333); - float _abs_337; - asm("abs.f32 %0, %1;" : "=f"(_abs_337) : "f"(rp_z_3.y)); - float _neg_336; - asm("neg.f32 %0, %1;" : "=f"(_neg_336) : "f"(_abs_337)); - float _exp_335 = __nv_expf(_neg_336); - float2 rp_e_3 = make_float2(_exp_332, _exp_335); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_335), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_338; + asm("abs.f32 %0, %1;" : "=f"(_abs_338) : "f"(rp_z_3.x)); + float _neg_337; + asm("neg.f32 %0, %1;" : "=f"(_neg_337) : "f"(_abs_338)); + float _exp_336 = __nv_expf(_neg_337); + float _abs_341; + asm("abs.f32 %0, %1;" : "=f"(_abs_341) : "f"(rp_z_3.y)); + float _neg_340; + asm("neg.f32 %0, %1;" : "=f"(_neg_340) : "f"(_abs_341)); + float _exp_339 = __nv_expf(_neg_340); + float2 rp_e_3 = make_float2(_exp_336, _exp_339); float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_338 = approx_rcp(rp_den_3.x); - float _rcp_339 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_340 = make_float2(_rcp_338, _rcp_339); + float _rcp_342 = approx_rcp(rp_den_3.x); + float _rcp_343 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_344 = make_float2(_rcp_342, _rcp_343); float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_340)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_344)); float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_341 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_345 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_345)); u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_342 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_346 = make_float2(gacc[4], gacc[4]); float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_342), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_345; - asm("abs.f32 %0, %1;" : "=f"(_abs_345) : "f"(rp_z_4.x)); - float _neg_344; - asm("neg.f32 %0, %1;" : "=f"(_neg_344) : "f"(_abs_345)); - float _exp_343 = __nv_expf(_neg_344); - float _abs_348; - asm("abs.f32 %0, %1;" : "=f"(_abs_348) : "f"(rp_z_4.y)); - float _neg_347; - asm("neg.f32 %0, %1;" : "=f"(_neg_347) : "f"(_abs_348)); - float _exp_346 = __nv_expf(_neg_347); - float2 rp_e_4 = make_float2(_exp_343, _exp_346); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_346), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_349; + asm("abs.f32 %0, %1;" : "=f"(_abs_349) : "f"(rp_z_4.x)); + float _neg_348; + asm("neg.f32 %0, %1;" : "=f"(_neg_348) : "f"(_abs_349)); + float _exp_347 = __nv_expf(_neg_348); + float _abs_352; + asm("abs.f32 %0, %1;" : "=f"(_abs_352) : "f"(rp_z_4.y)); + float _neg_351; + asm("neg.f32 %0, %1;" : "=f"(_neg_351) : "f"(_abs_352)); + float _exp_350 = __nv_expf(_neg_351); + float2 rp_e_4 = make_float2(_exp_347, _exp_350); float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_349 = approx_rcp(rp_den_4.x); - float _rcp_350 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_351 = make_float2(_rcp_349, _rcp_350); + float _rcp_353 = approx_rcp(rp_den_4.x); + float _rcp_354 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_355 = make_float2(_rcp_353, _rcp_354); float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_351)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_355)); float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_352 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_356 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_352)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_356)); u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_353 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_357 = make_float2(gacc[4], gacc[4]); float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_356; - asm("abs.f32 %0, %1;" : "=f"(_abs_356) : "f"(rp_z_5.x)); - float _neg_355; - asm("neg.f32 %0, %1;" : "=f"(_neg_355) : "f"(_abs_356)); - float _exp_354 = __nv_expf(_neg_355); - float _abs_359; - asm("abs.f32 %0, %1;" : "=f"(_abs_359) : "f"(rp_z_5.y)); - float _neg_358; - asm("neg.f32 %0, %1;" : "=f"(_neg_358) : "f"(_abs_359)); - float _exp_357 = __nv_expf(_neg_358); - float2 rp_e_5 = make_float2(_exp_354, _exp_357); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_357), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_360; + asm("abs.f32 %0, %1;" : "=f"(_abs_360) : "f"(rp_z_5.x)); + float _neg_359; + asm("neg.f32 %0, %1;" : "=f"(_neg_359) : "f"(_abs_360)); + float _exp_358 = __nv_expf(_neg_359); + float _abs_363; + asm("abs.f32 %0, %1;" : "=f"(_abs_363) : "f"(rp_z_5.y)); + float _neg_362; + asm("neg.f32 %0, %1;" : "=f"(_neg_362) : "f"(_abs_363)); + float _exp_361 = __nv_expf(_neg_362); + float2 rp_e_5 = make_float2(_exp_358, _exp_361); float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_360 = approx_rcp(rp_den_5.x); - float _rcp_361 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_362 = make_float2(_rcp_360, _rcp_361); + float _rcp_364 = approx_rcp(rp_den_5.x); + float _rcp_365 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_366 = make_float2(_rcp_364, _rcp_365); float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_362)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_366)); float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_363 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_367 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_363)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_367)); u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_364 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_368 = make_float2(gacc[4], gacc[4]); float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_364), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_367; - asm("abs.f32 %0, %1;" : "=f"(_abs_367) : "f"(rp_z_6.x)); - float _neg_366; - asm("neg.f32 %0, %1;" : "=f"(_neg_366) : "f"(_abs_367)); - float _exp_365 = __nv_expf(_neg_366); - float _abs_370; - asm("abs.f32 %0, %1;" : "=f"(_abs_370) : "f"(rp_z_6.y)); - float _neg_369; - asm("neg.f32 %0, %1;" : "=f"(_neg_369) : "f"(_abs_370)); - float _exp_368 = __nv_expf(_neg_369); - float2 rp_e_6 = make_float2(_exp_365, _exp_368); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_368), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_371; + asm("abs.f32 %0, %1;" : "=f"(_abs_371) : "f"(rp_z_6.x)); + float _neg_370; + asm("neg.f32 %0, %1;" : "=f"(_neg_370) : "f"(_abs_371)); + float _exp_369 = __nv_expf(_neg_370); + float _abs_374; + asm("abs.f32 %0, %1;" : "=f"(_abs_374) : "f"(rp_z_6.y)); + float _neg_373; + asm("neg.f32 %0, %1;" : "=f"(_neg_373) : "f"(_abs_374)); + float _exp_372 = __nv_expf(_neg_373); + float2 rp_e_6 = make_float2(_exp_369, _exp_372); float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_371 = approx_rcp(rp_den_6.x); - float _rcp_372 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_373 = make_float2(_rcp_371, _rcp_372); + float _rcp_375 = approx_rcp(rp_den_6.x); + float _rcp_376 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_377 = make_float2(_rcp_375, _rcp_376); float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_373)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_377)); float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_374 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_378 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_374)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_378)); u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4292,54 +3867,54 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_375; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_375) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_375; + uint32_t _prmt_379; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_379; int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_376 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_376)); - *(reinterpret_cast(_gst_376) + (0)) = pq_4[0]; + auto _gst_380 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = pq_4[0]; } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_377; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_377) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_377; + uint32_t _prmt_381; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_381; int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_378 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_378)); - *(reinterpret_cast(_gst_378) + (0)) = pq_5[0]; + auto _gst_382 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = pq_5[0]; } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_379; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_379; + uint32_t _prmt_383; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_383) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_383; int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_380 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_380)); - *(reinterpret_cast(_gst_380) + (0)) = pq_6[0]; + auto _gst_384 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = pq_6[0]; } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_381; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_381; + uint32_t _prmt_385; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_385) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_385; int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_382 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_382)); - *(reinterpret_cast(_gst_382) + (0)) = pq_7[0]; + auto _gst_386 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_386)); + *(reinterpret_cast(_gst_386) + (0)) = pq_7[0]; } } int t_a_8 = 32 + 8 * th; @@ -4404,47 +3979,47 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_383; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_383) : "f"(s_a_0), "f"(nm_t_1)); + float _sub_387; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_387) : "f"(s_a_0), "f"(nm_t_1)); float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_383), "f"(pa_0)); - float _sub_384; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_384) : "f"(pa_0), "f"(nm_t_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_387), "f"(pa_0)); + float _sub_388; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_388) : "f"(pa_0), "f"(nm_t_1)); float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_384), "f"(s_a_0)); - float _abs_386; - asm("abs.f32 %0, %1;" : "=f"(_abs_386) : "f"(s_a_0)); - float _abs_387; - asm("abs.f32 %0, %1;" : "=f"(_abs_387) : "f"(pa_0)); - uint32_t _setp_385_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_385_bits) : "f"(_abs_386), "f"(_abs_387)); - bool _setp_385 = _setp_385_bits != 0; - float nm_err_1 = ((_setp_385) ? nm_es_1 : nm_ex_1); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_388; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_388), "f"(s_a_0)); + float _abs_390; + asm("abs.f32 %0, %1;" : "=f"(_abs_390) : "f"(s_a_0)); + float _abs_391; + asm("abs.f32 %0, %1;" : "=f"(_abs_391) : "f"(pa_0)); + uint32_t _setp_389_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_389_bits) : "f"(_abs_390), "f"(_abs_391)); + bool _setp_389 = _setp_389_bits != 0; + float nm_err_1 = ((_setp_389) ? nm_es_1 : nm_ex_1); + float _add_392; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_392; s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_389; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_389) : "f"(s_b_0), "f"(nm_t_2)); + float _sub_393; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_393) : "f"(s_b_0), "f"(nm_t_2)); float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_389), "f"(pb_0)); - float _sub_390; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_390) : "f"(pb_0), "f"(nm_t_2)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_393), "f"(pb_0)); + float _sub_394; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_394) : "f"(pb_0), "f"(nm_t_2)); float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_390), "f"(s_b_0)); - float _abs_392; - asm("abs.f32 %0, %1;" : "=f"(_abs_392) : "f"(s_b_0)); - float _abs_393; - asm("abs.f32 %0, %1;" : "=f"(_abs_393) : "f"(pb_0)); - uint32_t _setp_391_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_391_bits) : "f"(_abs_392), "f"(_abs_393)); - bool _setp_391 = _setp_391_bits != 0; - float nm_err_2 = ((_setp_391) ? nm_es_2 : nm_ex_2); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_394; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_394), "f"(s_b_0)); + float _abs_396; + asm("abs.f32 %0, %1;" : "=f"(_abs_396) : "f"(s_b_0)); + float _abs_397; + asm("abs.f32 %0, %1;" : "=f"(_abs_397) : "f"(pb_0)); + uint32_t _setp_395_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_395_bits) : "f"(_abs_396), "f"(_abs_397)); + bool _setp_395 = _setp_395_bits != 0; + float nm_err_2 = ((_setp_395) ? nm_es_2 : nm_ex_2); + float _add_398; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_398; s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4452,8 +4027,8 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_395 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_395), "f"(off_acc[0]) : "memory"); + uint32_t _addr_399 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_399), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4479,9 +4054,9 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1 = r_297[0]; + float r_255[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1 = r_255[0]; float gc_2 = gn_1 - 21.640425f; int eb_1 = 81920 + th * -32768; int ew_1 = eb_1 >> 2; @@ -4490,368 +4065,408 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int gb8_1 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1 * 128); int kb8_1 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1 * 128); float dbp_1[8]; - float qo_1[8]; - tmem_ld_x8(&qo_1[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); - float ko_1[8]; - tmem_ld_x8(&ko_1[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); - float qdg_1[8]; - tmem_ld_x8(&qdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); - float kdg_1[8]; - tmem_ld_x8(&kdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); + float qo_2[8]; + tmem_ld_x8(&qo_2[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); + float ko_2[8]; + tmem_ld_x8(&ko_2[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); + float qdg_2[8]; + tmem_ld_x8(&qdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); + float kdg_2[8]; + tmem_ld_x8(&kdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); float dkt_2[8]; tmem_ld_x8(&dkt_2[0], taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th)); - float btv_1[8]; + float btv_2[8]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1[0])), "=r"(*reinterpret_cast(&btv_1[(0) + 1])), "=r"(*reinterpret_cast(&btv_1[(0) + 2])), "=r"(*reinterpret_cast(&btv_1[(0) + 3])) + : "=r"(*reinterpret_cast(&btv_2[0])), "=r"(*reinterpret_cast(&btv_2[(0) + 1])), "=r"(*reinterpret_cast(&btv_2[(0) + 2])), "=r"(*reinterpret_cast(&btv_2[(0) + 3])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1[4])), "=r"(*reinterpret_cast(&btv_1[(4) + 1])), "=r"(*reinterpret_cast(&btv_1[(4) + 2])), "=r"(*reinterpret_cast(&btv_1[(4) + 3])) + : "=r"(*reinterpret_cast(&btv_2[4])), "=r"(*reinterpret_cast(&btv_2[(4) + 1])), "=r"(*reinterpret_cast(&btv_2[(4) + 2])), "=r"(*reinterpret_cast(&btv_2[(4) + 3])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th) + 16))); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1[1]; s_run_1[0] = 0.0f; - int t_16 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1 + 2944 + d_lane]; - float bt_t_16 = btv_1[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1); - float e_dg_16 = approx_exp2(g_t_16 - gc_2); - float dq2_16 = __fmaf_rn(qo_1[7], e_off_16, qdg_1[7] * e_dg_16); - float dk2_16 = __fmaf_rn(ko_1[7], e_off_16, kdg_1[7] * e_dg_16); - dbp_1[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + dkt_2[7]; - float dg_o_16 = __fmaf_rn(q_t_1_16, dq2_16, (dk2b_16 - dkt_2[7]) * k_t_16) + dgf_16; - s_run_1[0] = s_run_1[0] + dg_o_16; + int t_0 = 16 + 8 * th + 7; + float r_256[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_0 = r_256[0]; + unsigned int r_257[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_0 = __uint_as_float(r_257[0] << 16); + unsigned int r_258[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_0 = __uint_as_float(r_258[0] << 16); + float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); + float bt_t_0 = btv_2[7]; + float e_off_0 = approx_exp2(g_t_0 - gn_1); + float e_dg_0 = approx_exp2(g_t_0 - gc_2); + float dq2_0 = __fmaf_rn(qo_2[7], e_off_0, qdg_2[7] * e_dg_0); + float dk2_0 = __fmaf_rn(ko_2[7], e_off_0, kdg_2[7] * e_dg_0); + dbp_1[7] = dk2_0 * k_t_0; + float dk2b_0 = dk2_0 * bt_t_0; + float qe_2 = q_t_1_0 * e_dg_0; + float kp_13 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_13; + float kbe_2 = (float)kb_2 * e_dg_0; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg_2[7], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_13 * e_dg_0) * kdg_2[7]); + float dqf_0 = pool_f32[ew_1 + 896 + d_lane]; + float dkf_0 = pool_f32[ew_1 + 1920 + d_lane]; + float dgf_0 = pool_f32[ew_1 + 2944 + d_lane]; + float dq_o_0 = dq2_0 + dqf_0; + float dk_o_0 = dk2b_0 + dkf_0 + dkt_2[7]; + float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_2[7], k_t_0, cr_2)) + dgf_0; + s_run_1[0] = s_run_1[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_16 = t_16 * qrs; - if (t_16 < clen) { + int qoff_0 = t_0 * qrs; + if (t_0 < clen) { { - auto _gst_396 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_396)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_396) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_400 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_400)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_397 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_397)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_397) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_401 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_401)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_0; } } - int t_17 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1 + 2816 + d_lane]; - float bt_t_17 = btv_1[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1); - float e_dg_17 = approx_exp2(g_t_17 - gc_2); - float dq2_17 = __fmaf_rn(qo_1[6], e_off_17, qdg_1[6] * e_dg_17); - float dk2_17 = __fmaf_rn(ko_1[6], e_off_17, kdg_1[6] * e_dg_17); - dbp_1[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + dkt_2[6]; - float dg_o_17 = __fmaf_rn(q_t_1_17, dq2_17, (dk2b_17 - dkt_2[6]) * k_t_17) + dgf_17; - s_run_1[0] = s_run_1[0] + dg_o_17; + int t_1_1 = 16 + 8 * th + 6; + float r_259[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1_1 = r_259[0]; + unsigned int r_260[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1_1 = __uint_as_float(r_260[0] << 16); + unsigned int r_261[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1_2 = __uint_as_float(r_261[0] << 16); + float q_t_1_1_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_2); + float bt_t_1_1 = btv_2[6]; + float e_off_1_1 = approx_exp2(g_t_1_1 - gn_1); + float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_2); + float dq2_1_1 = __fmaf_rn(qo_2[6], e_off_1_1, qdg_2[6] * e_dg_1_1); + float dk2_1_1 = __fmaf_rn(ko_2[6], e_off_1_1, kdg_2[6] * e_dg_1_1); + dbp_1[6] = dk2_1_1 * k_t_1_1; + float dk2b_1_1 = dk2_1_1 * bt_t_1_1; + float qe_0 = q_t_1_1_1 * e_dg_1_1; + float kp_12_1 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12_1; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg_2[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12_1 * e_dg_1_1) * kdg_2[6]); + float dqf_1_1 = pool_f32[ew_1 + 768 + d_lane]; + float dkf_1_1 = pool_f32[ew_1 + 1792 + d_lane]; + float dgf_1_1 = pool_f32[ew_1 + 2816 + d_lane]; + float dq_o_1_1 = dq2_1_1 + dqf_1_1; + float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + dkt_2[6]; + float dg_o_1_1 = __fmaf_rn(q_t_1_1_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - dkt_2[6], k_t_1_1, cr_0)) + dgf_1_1; + s_run_1[0] = s_run_1[0] + dg_o_1_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_17 = t_17 * qrs; - if (t_17 < clen) { + int qoff_1_1 = t_1_1 * qrs; + if (t_1_1 < clen) { { - auto _gst_398 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_398)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_398) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_402 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_402)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_1_1; } { - auto _gst_399 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_399)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_399) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_403 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_403)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_1_1; } } - int t_18 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1 + 2688 + d_lane]; - float bt_t_18 = btv_1[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1); - float e_dg_18 = approx_exp2(g_t_18 - gc_2); - float dq2_18 = __fmaf_rn(qo_1[5], e_off_18, qdg_1[5] * e_dg_18); - float dk2_18 = __fmaf_rn(ko_1[5], e_off_18, kdg_1[5] * e_dg_18); - dbp_1[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + dkt_2[5]; - float dg_o_18 = __fmaf_rn(q_t_1_18, dq2_18, (dk2b_18 - dkt_2[5]) * k_t_18) + dgf_18; - s_run_1[0] = s_run_1[0] + dg_o_18; + int t_2 = 16 + 8 * th + 5; + float r_262[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_262[0]; + unsigned int r_263[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_263[0] << 16); + unsigned int r_264[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2_1 = __uint_as_float(r_264[0] << 16); + float q_t_1_2_1 = ((t_2 >= clen) ? 0.0f : q_t_2_1); + float bt_t_2 = btv_2[5]; + float e_off_2 = approx_exp2(g_t_2 - gn_1); + float e_dg_2 = approx_exp2(g_t_2 - gc_2); + float dq2_2 = __fmaf_rn(qo_2[5], e_off_2, qdg_2[5] * e_dg_2); + float dk2_2 = __fmaf_rn(ko_2[5], e_off_2, kdg_2[5] * e_dg_2); + dbp_1[5] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float qe_1_1 = q_t_1_2_1 * e_dg_2; + float kp_13_1 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1_1 = (__nv_bfloat16)kp_13_1; + float kbe_1_1 = (float)kb_1_1 * e_dg_2; + float cr_1_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1_1) & 4294959104u) - qe_1_1, qdg_2[5], (__uint_as_float(__as_u32(kbe_1_1) & 4294959104u) - kp_13_1 * e_dg_2) * kdg_2[5]); + float dqf_2 = pool_f32[ew_1 + 640 + d_lane]; + float dkf_2 = pool_f32[ew_1 + 1664 + d_lane]; + float dgf_2 = pool_f32[ew_1 + 2688 + d_lane]; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + dkt_2[5]; + float dg_o_2 = __fmaf_rn(q_t_1_2_1, dq2_2, __fmaf_rn(dk2b_2 - dkt_2[5], k_t_2, cr_1_1)) + dgf_2; + s_run_1[0] = s_run_1[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_18 = t_18 * qrs; - if (t_18 < clen) { + int qoff_2 = t_2 * qrs; + if (t_2 < clen) { { - auto _gst_400 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_400)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_404 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_404)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_401 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_401)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_405 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_405)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_2; } } - int t_19 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1 + 2560 + d_lane]; - float bt_t_19 = btv_1[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1); - float e_dg_19 = approx_exp2(g_t_19 - gc_2); - float dq2_19 = __fmaf_rn(qo_1[4], e_off_19, qdg_1[4] * e_dg_19); - float dk2_19 = __fmaf_rn(ko_1[4], e_off_19, kdg_1[4] * e_dg_19); - dbp_1[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + dkt_2[4]; - float dg_o_19 = __fmaf_rn(q_t_1_19, dq2_19, (dk2b_19 - dkt_2[4]) * k_t_19) + dgf_19; - s_run_1[0] = s_run_1[0] + dg_o_19; + int t_3 = 16 + 8 * th + 4; + float r_265[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_3 = r_265[0]; + unsigned int r_266[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_3 = __uint_as_float(r_266[0] << 16); + unsigned int r_267[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_3 = __uint_as_float(r_267[0] << 16); + float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); + float bt_t_3 = btv_2[4]; + float e_off_3 = approx_exp2(g_t_3 - gn_1); + float e_dg_3 = approx_exp2(g_t_3 - gc_2); + float dq2_3 = __fmaf_rn(qo_2[4], e_off_3, qdg_2[4] * e_dg_3); + float dk2_3 = __fmaf_rn(ko_2[4], e_off_3, kdg_2[4] * e_dg_3); + dbp_1[4] = dk2_3 * k_t_3; + float dk2b_3 = dk2_3 * bt_t_3; + float qe_2_1 = q_t_1_3 * e_dg_3; + float kp_14 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2_1 = (__nv_bfloat16)kp_14; + float kbe_2_1 = (float)kb_2_1 * e_dg_3; + float cr_2_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_2_1) & 4294959104u) - qe_2_1, qdg_2[4], (__uint_as_float(__as_u32(kbe_2_1) & 4294959104u) - kp_14 * e_dg_3) * kdg_2[4]); + float dqf_3 = pool_f32[ew_1 + 512 + d_lane]; + float dkf_3 = pool_f32[ew_1 + 1536 + d_lane]; + float dgf_3 = pool_f32[ew_1 + 2560 + d_lane]; + float dq_o_3 = dq2_3 + dqf_3; + float dk_o_3 = dk2b_3 + dkf_3 + dkt_2[4]; + float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_2[4], k_t_3, cr_2_1)) + dgf_3; + s_run_1[0] = s_run_1[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_19 = t_19 * qrs; - if (t_19 < clen) { + int qoff_3 = t_3 * qrs; + if (t_3 < clen) { { - auto _gst_402 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_402)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_406 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_406)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_403 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_403)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_407 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_407)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_3; } } - int t_20 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1 + 2432 + d_lane]; - float bt_t_20 = btv_1[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1); - float e_dg_20 = approx_exp2(g_t_20 - gc_2); - float dq2_20 = __fmaf_rn(qo_1[3], e_off_20, qdg_1[3] * e_dg_20); - float dk2_20 = __fmaf_rn(ko_1[3], e_off_20, kdg_1[3] * e_dg_20); - dbp_1[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + dkt_2[3]; - float dg_o_20 = __fmaf_rn(q_t_1_20, dq2_20, (dk2b_20 - dkt_2[3]) * k_t_20) + dgf_20; - s_run_1[0] = s_run_1[0] + dg_o_20; + int t_4 = 16 + 8 * th + 3; + float r_268[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_4 = r_268[0]; + unsigned int r_269[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_4 = __uint_as_float(r_269[0] << 16); + unsigned int r_270[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_4 = __uint_as_float(r_270[0] << 16); + float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); + float bt_t_4 = btv_2[3]; + float e_off_4 = approx_exp2(g_t_4 - gn_1); + float e_dg_4 = approx_exp2(g_t_4 - gc_2); + float dq2_4 = __fmaf_rn(qo_2[3], e_off_4, qdg_2[3] * e_dg_4); + float dk2_4 = __fmaf_rn(ko_2[3], e_off_4, kdg_2[3] * e_dg_4); + dbp_1[3] = dk2_4 * k_t_4; + float dk2b_4 = dk2_4 * bt_t_4; + float qe_3 = q_t_1_4 * e_dg_4; + float kp_15 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg_2[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_4) * kdg_2[3]); + float dqf_4 = pool_f32[ew_1 + 384 + d_lane]; + float dkf_4 = pool_f32[ew_1 + 1408 + d_lane]; + float dgf_4 = pool_f32[ew_1 + 2432 + d_lane]; + float dq_o_4 = dq2_4 + dqf_4; + float dk_o_4 = dk2b_4 + dkf_4 + dkt_2[3]; + float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_2[3], k_t_4, cr_3)) + dgf_4; + s_run_1[0] = s_run_1[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_20 = t_20 * qrs; - if (t_20 < clen) { + int qoff_4 = t_4 * qrs; + if (t_4 < clen) { { - auto _gst_404 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_404)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_408 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_408)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_405 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_405)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_409 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_409)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_4; } } - int t_21 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1 + 2304 + d_lane]; - float bt_t_21 = btv_1[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1); - float e_dg_21 = approx_exp2(g_t_21 - gc_2); - float dq2_21 = __fmaf_rn(qo_1[2], e_off_21, qdg_1[2] * e_dg_21); - float dk2_21 = __fmaf_rn(ko_1[2], e_off_21, kdg_1[2] * e_dg_21); - dbp_1[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + dkt_2[2]; - float dg_o_21 = __fmaf_rn(q_t_1_21, dq2_21, (dk2b_21 - dkt_2[2]) * k_t_21) + dgf_21; - s_run_1[0] = s_run_1[0] + dg_o_21; + int t_5 = 16 + 8 * th + 2; + float r_271[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_5 = r_271[0]; + unsigned int r_272[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_5 = __uint_as_float(r_272[0] << 16); + unsigned int r_273[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_5 = __uint_as_float(r_273[0] << 16); + float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); + float bt_t_5 = btv_2[2]; + float e_off_5 = approx_exp2(g_t_5 - gn_1); + float e_dg_5 = approx_exp2(g_t_5 - gc_2); + float dq2_5 = __fmaf_rn(qo_2[2], e_off_5, qdg_2[2] * e_dg_5); + float dk2_5 = __fmaf_rn(ko_2[2], e_off_5, kdg_2[2] * e_dg_5); + dbp_1[2] = dk2_5 * k_t_5; + float dk2b_5 = dk2_5 * bt_t_5; + float qe_4 = q_t_1_5 * e_dg_5; + float kp_16 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg_2[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_5) * kdg_2[2]); + float dqf_5 = pool_f32[ew_1 + 256 + d_lane]; + float dkf_5 = pool_f32[ew_1 + 1280 + d_lane]; + float dgf_5 = pool_f32[ew_1 + 2304 + d_lane]; + float dq_o_5 = dq2_5 + dqf_5; + float dk_o_5 = dk2b_5 + dkf_5 + dkt_2[2]; + float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_2[2], k_t_5, cr_4)) + dgf_5; + s_run_1[0] = s_run_1[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_21 = t_21 * qrs; - if (t_21 < clen) { + int qoff_5 = t_5 * qrs; + if (t_5 < clen) { { - auto _gst_406 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_406)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_410 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_410)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_407 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_407)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_411 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_411)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_5; } } - int t_22 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1 + 2176 + d_lane]; - float bt_t_22 = btv_1[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1); - float e_dg_22 = approx_exp2(g_t_22 - gc_2); - float dq2_22 = __fmaf_rn(qo_1[1], e_off_22, qdg_1[1] * e_dg_22); - float dk2_22 = __fmaf_rn(ko_1[1], e_off_22, kdg_1[1] * e_dg_22); - dbp_1[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + dkt_2[1]; - float dg_o_22 = __fmaf_rn(q_t_1_22, dq2_22, (dk2b_22 - dkt_2[1]) * k_t_22) + dgf_22; - s_run_1[0] = s_run_1[0] + dg_o_22; + int t_6 = 16 + 8 * th + 1; + float r_274[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_6 = r_274[0]; + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_6 = __uint_as_float(r_275[0] << 16); + unsigned int r_276[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_6 = __uint_as_float(r_276[0] << 16); + float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); + float bt_t_6 = btv_2[1]; + float e_off_6 = approx_exp2(g_t_6 - gn_1); + float e_dg_6 = approx_exp2(g_t_6 - gc_2); + float dq2_6 = __fmaf_rn(qo_2[1], e_off_6, qdg_2[1] * e_dg_6); + float dk2_6 = __fmaf_rn(ko_2[1], e_off_6, kdg_2[1] * e_dg_6); + dbp_1[1] = dk2_6 * k_t_6; + float dk2b_6 = dk2_6 * bt_t_6; + float qe_5 = q_t_1_6 * e_dg_6; + float kp_17 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg_2[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_6) * kdg_2[1]); + float dqf_6 = pool_f32[ew_1 + 128 + d_lane]; + float dkf_6 = pool_f32[ew_1 + 1152 + d_lane]; + float dgf_6 = pool_f32[ew_1 + 2176 + d_lane]; + float dq_o_6 = dq2_6 + dqf_6; + float dk_o_6 = dk2b_6 + dkf_6 + dkt_2[1]; + float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_2[1], k_t_6, cr_5)) + dgf_6; + s_run_1[0] = s_run_1[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_22 = t_22 * qrs; - if (t_22 < clen) { + int qoff_6 = t_6 * qrs; + if (t_6 < clen) { { - auto _gst_408 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_408)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_412 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_412)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_412) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_409 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_409)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_413 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_413)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_413) + (0)) = (__nv_bfloat16)dk_o_6; } } - int t_23 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1 + d_lane]; - float dkf_23 = pool_f32[ew_1 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1 + 2048 + d_lane]; - float bt_t_23 = btv_1[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1); - float e_dg_23 = approx_exp2(g_t_23 - gc_2); - float dq2_23 = __fmaf_rn(qo_1[0], e_off_23, qdg_1[0] * e_dg_23); - float dk2_23 = __fmaf_rn(ko_1[0], e_off_23, kdg_1[0] * e_dg_23); - dbp_1[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + dkt_2[0]; - float dg_o_23 = __fmaf_rn(q_t_1_23, dq2_23, (dk2b_23 - dkt_2[0]) * k_t_23) + dgf_23; - s_run_1[0] = s_run_1[0] + dg_o_23; + int t_7 = 16 + 8 * th; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_7 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_7 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_7 = __uint_as_float(r_279[0] << 16); + float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); + float bt_t_7 = btv_2[0]; + float e_off_7 = approx_exp2(g_t_7 - gn_1); + float e_dg_7 = approx_exp2(g_t_7 - gc_2); + float dq2_7 = __fmaf_rn(qo_2[0], e_off_7, qdg_2[0] * e_dg_7); + float dk2_7 = __fmaf_rn(ko_2[0], e_off_7, kdg_2[0] * e_dg_7); + dbp_1[0] = dk2_7 * k_t_7; + float dk2b_7 = dk2_7 * bt_t_7; + float qe_6 = q_t_1_7 * e_dg_7; + float kp_18 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg_2[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_7) * kdg_2[0]); + float dqf_7 = pool_f32[ew_1 + d_lane]; + float dkf_7 = pool_f32[ew_1 + 1024 + d_lane]; + float dgf_7 = pool_f32[ew_1 + 2048 + d_lane]; + float dq_o_7 = dq2_7 + dqf_7; + float dk_o_7 = dk2b_7 + dkf_7 + dkt_2[0]; + float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_2[0], k_t_7, cr_6)) + dgf_7; + s_run_1[0] = s_run_1[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_23 = t_23 * qrs; - if (t_23 < clen) { + int qoff_7 = t_7 * qrs; + if (t_7 < clen) { { - auto _gst_410 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_410)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_414 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_414)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_414) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_411 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_411)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_415 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_415)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_415) + (0)) = (__nv_bfloat16)dk_o_7; } } float tot_1 = s_run_1[0]; @@ -4860,39 +4475,39 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_412 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_412; + float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_416; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_413; + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_417; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_414 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_414; + float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_418; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_415; + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_419; float v2_17[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_17[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_416; + float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_17[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_420; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_17[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_417; + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_17[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_421; float send_19 = ((b2_1 == 0) ? v2_17[1] : v2_17[0]); - float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_418; - float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_419; - float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_420; + float _shfl_xor_sync_422 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_422; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_423; + float _shfl_xor_sync_424 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_424; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_421 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_421), "f"(tot_1) : "memory"); + uint32_t _addr_425 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_425), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4924,132 +4539,132 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_422 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_426 = make_float2(gacc[4], gacc[4]); float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_422), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_425; - asm("abs.f32 %0, %1;" : "=f"(_abs_425) : "f"(rp_z_7.x)); - float _neg_424; - asm("neg.f32 %0, %1;" : "=f"(_neg_424) : "f"(_abs_425)); - float _exp_423 = __nv_expf(_neg_424); - float _abs_428; - asm("abs.f32 %0, %1;" : "=f"(_abs_428) : "f"(rp_z_7.y)); - float _neg_427; - asm("neg.f32 %0, %1;" : "=f"(_neg_427) : "f"(_abs_428)); - float _exp_426 = __nv_expf(_neg_427); - float2 rp_e_7 = make_float2(_exp_423, _exp_426); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_426), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_429; + asm("abs.f32 %0, %1;" : "=f"(_abs_429) : "f"(rp_z_7.x)); + float _neg_428; + asm("neg.f32 %0, %1;" : "=f"(_neg_428) : "f"(_abs_429)); + float _exp_427 = __nv_expf(_neg_428); + float _abs_432; + asm("abs.f32 %0, %1;" : "=f"(_abs_432) : "f"(rp_z_7.y)); + float _neg_431; + asm("neg.f32 %0, %1;" : "=f"(_neg_431) : "f"(_abs_432)); + float _exp_430 = __nv_expf(_neg_431); + float2 rp_e_7 = make_float2(_exp_427, _exp_430); float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_429 = approx_rcp(rp_den_7.x); - float _rcp_430 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_431 = make_float2(_rcp_429, _rcp_430); + float _rcp_433 = approx_rcp(rp_den_7.x); + float _rcp_434 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_435 = make_float2(_rcp_433, _rcp_434); float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_431)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_435)); float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_432 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_436 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_432)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_436)); u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_433 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_437 = make_float2(gacc[4], gacc[4]); float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_433), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_436; - asm("abs.f32 %0, %1;" : "=f"(_abs_436) : "f"(rp_z_8.x)); - float _neg_435; - asm("neg.f32 %0, %1;" : "=f"(_neg_435) : "f"(_abs_436)); - float _exp_434 = __nv_expf(_neg_435); - float _abs_439; - asm("abs.f32 %0, %1;" : "=f"(_abs_439) : "f"(rp_z_8.y)); - float _neg_438; - asm("neg.f32 %0, %1;" : "=f"(_neg_438) : "f"(_abs_439)); - float _exp_437 = __nv_expf(_neg_438); - float2 rp_e_8 = make_float2(_exp_434, _exp_437); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_437), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_440; + asm("abs.f32 %0, %1;" : "=f"(_abs_440) : "f"(rp_z_8.x)); + float _neg_439; + asm("neg.f32 %0, %1;" : "=f"(_neg_439) : "f"(_abs_440)); + float _exp_438 = __nv_expf(_neg_439); + float _abs_443; + asm("abs.f32 %0, %1;" : "=f"(_abs_443) : "f"(rp_z_8.y)); + float _neg_442; + asm("neg.f32 %0, %1;" : "=f"(_neg_442) : "f"(_abs_443)); + float _exp_441 = __nv_expf(_neg_442); + float2 rp_e_8 = make_float2(_exp_438, _exp_441); float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_440 = approx_rcp(rp_den_8.x); - float _rcp_441 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_442 = make_float2(_rcp_440, _rcp_441); + float _rcp_444 = approx_rcp(rp_den_8.x); + float _rcp_445 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_446 = make_float2(_rcp_444, _rcp_445); float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_442)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_446)); float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_443 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_447 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_443)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_447)); u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_444 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_448 = make_float2(gacc[4], gacc[4]); float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_444), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_447; - asm("abs.f32 %0, %1;" : "=f"(_abs_447) : "f"(rp_z_9.x)); - float _neg_446; - asm("neg.f32 %0, %1;" : "=f"(_neg_446) : "f"(_abs_447)); - float _exp_445 = __nv_expf(_neg_446); - float _abs_450; - asm("abs.f32 %0, %1;" : "=f"(_abs_450) : "f"(rp_z_9.y)); - float _neg_449; - asm("neg.f32 %0, %1;" : "=f"(_neg_449) : "f"(_abs_450)); - float _exp_448 = __nv_expf(_neg_449); - float2 rp_e_9 = make_float2(_exp_445, _exp_448); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_448), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_451; + asm("abs.f32 %0, %1;" : "=f"(_abs_451) : "f"(rp_z_9.x)); + float _neg_450; + asm("neg.f32 %0, %1;" : "=f"(_neg_450) : "f"(_abs_451)); + float _exp_449 = __nv_expf(_neg_450); + float _abs_454; + asm("abs.f32 %0, %1;" : "=f"(_abs_454) : "f"(rp_z_9.y)); + float _neg_453; + asm("neg.f32 %0, %1;" : "=f"(_neg_453) : "f"(_abs_454)); + float _exp_452 = __nv_expf(_neg_453); + float2 rp_e_9 = make_float2(_exp_449, _exp_452); float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_451 = approx_rcp(rp_den_9.x); - float _rcp_452 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_453 = make_float2(_rcp_451, _rcp_452); + float _rcp_455 = approx_rcp(rp_den_9.x); + float _rcp_456 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_457 = make_float2(_rcp_455, _rcp_456); float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_453)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_457)); float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_454 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_458 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_454)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_455 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_459 = make_float2(gacc[4], gacc[4]); float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_455), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_458; - asm("abs.f32 %0, %1;" : "=f"(_abs_458) : "f"(rp_z_10.x)); - float _neg_457; - asm("neg.f32 %0, %1;" : "=f"(_neg_457) : "f"(_abs_458)); - float _exp_456 = __nv_expf(_neg_457); - float _abs_461; - asm("abs.f32 %0, %1;" : "=f"(_abs_461) : "f"(rp_z_10.y)); - float _neg_460; - asm("neg.f32 %0, %1;" : "=f"(_neg_460) : "f"(_abs_461)); - float _exp_459 = __nv_expf(_neg_460); - float2 rp_e_10 = make_float2(_exp_456, _exp_459); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_459), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_462; + asm("abs.f32 %0, %1;" : "=f"(_abs_462) : "f"(rp_z_10.x)); + float _neg_461; + asm("neg.f32 %0, %1;" : "=f"(_neg_461) : "f"(_abs_462)); + float _exp_460 = __nv_expf(_neg_461); + float _abs_465; + asm("abs.f32 %0, %1;" : "=f"(_abs_465) : "f"(rp_z_10.y)); + float _neg_464; + asm("neg.f32 %0, %1;" : "=f"(_neg_464) : "f"(_abs_465)); + float _exp_463 = __nv_expf(_neg_464); + float2 rp_e_10 = make_float2(_exp_460, _exp_463); float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_462 = approx_rcp(rp_den_10.x); - float _rcp_463 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_464 = make_float2(_rcp_462, _rcp_463); + float _rcp_466 = approx_rcp(rp_den_10.x); + float _rcp_467 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_468 = make_float2(_rcp_466, _rcp_467); float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_464)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_468)); float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_465 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_469 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_465)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -5064,54 +4679,54 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_466; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_466) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_466; + uint32_t _prmt_470; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_470; int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_467 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_467)); - *(reinterpret_cast(_gst_467) + (0)) = pq_8[0]; + auto _gst_471 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_471)); + *(reinterpret_cast(_gst_471) + (0)) = pq_8[0]; } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_468; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_468) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_468; + uint32_t _prmt_472; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_472; int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_469 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_469)); - *(reinterpret_cast(_gst_469) + (0)) = pq_9[0]; + auto _gst_473 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_473)); + *(reinterpret_cast(_gst_473) + (0)) = pq_9[0]; } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_470; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_470; + uint32_t _prmt_474; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_474) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_474; int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_471 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_471)); - *(reinterpret_cast(_gst_471) + (0)) = pq_10[0]; + auto _gst_475 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_475)); + *(reinterpret_cast(_gst_475) + (0)) = pq_10[0]; } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_472; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_472; + uint32_t _prmt_476; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_476) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_476; int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_473 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_473)); - *(reinterpret_cast(_gst_473) + (0)) = pq_11[0]; + auto _gst_477 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_477)); + *(reinterpret_cast(_gst_477) + (0)) = pq_11[0]; } } int t_a_16 = 16 + 8 * th; @@ -5176,47 +4791,47 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_474; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_474) : "f"(s_a_1), "f"(nm_t_3)); + float _sub_478; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_478) : "f"(s_a_1), "f"(nm_t_3)); float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_474), "f"(pa_1)); - float _sub_475; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_475) : "f"(pa_1), "f"(nm_t_3)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_478), "f"(pa_1)); + float _sub_479; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_479) : "f"(pa_1), "f"(nm_t_3)); float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_475), "f"(s_a_1)); - float _abs_477; - asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(s_a_1)); - float _abs_478; - asm("abs.f32 %0, %1;" : "=f"(_abs_478) : "f"(pa_1)); - uint32_t _setp_476_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_476_bits) : "f"(_abs_477), "f"(_abs_478)); - bool _setp_476 = _setp_476_bits != 0; - float nm_err_3 = ((_setp_476) ? nm_es_3 : nm_ex_3); - float _add_479; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_479) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_479; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_479), "f"(s_a_1)); + float _abs_481; + asm("abs.f32 %0, %1;" : "=f"(_abs_481) : "f"(s_a_1)); + float _abs_482; + asm("abs.f32 %0, %1;" : "=f"(_abs_482) : "f"(pa_1)); + uint32_t _setp_480_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_480_bits) : "f"(_abs_481), "f"(_abs_482)); + bool _setp_480 = _setp_480_bits != 0; + float nm_err_3 = ((_setp_480) ? nm_es_3 : nm_ex_3); + float _add_483; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_483) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_483; s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_480; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_480) : "f"(s_b_1), "f"(nm_t_4)); + float _sub_484; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_484) : "f"(s_b_1), "f"(nm_t_4)); float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_480), "f"(pb_1)); - float _sub_481; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_481) : "f"(pb_1), "f"(nm_t_4)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_484), "f"(pb_1)); + float _sub_485; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_485) : "f"(pb_1), "f"(nm_t_4)); float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_481), "f"(s_b_1)); - float _abs_483; - asm("abs.f32 %0, %1;" : "=f"(_abs_483) : "f"(s_b_1)); - float _abs_484; - asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(pb_1)); - uint32_t _setp_482_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_482_bits) : "f"(_abs_483), "f"(_abs_484)); - bool _setp_482 = _setp_482_bits != 0; - float nm_err_4 = ((_setp_482) ? nm_es_4 : nm_ex_4); - float _add_485; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_485) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_485; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_485), "f"(s_b_1)); + float _abs_487; + asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(s_b_1)); + float _abs_488; + asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(pb_1)); + uint32_t _setp_486_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_486_bits) : "f"(_abs_487), "f"(_abs_488)); + bool _setp_486 = _setp_486_bits != 0; + float nm_err_4 = ((_setp_486) ? nm_es_4 : nm_ex_4); + float _add_489; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_489) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_489; s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5224,8 +4839,8 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_486 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_486), "f"(off_acc[0]) : "memory"); + uint32_t _addr_490 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_490), "f"(off_acc[0]) : "memory"); } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5249,9 +4864,9 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_2 = r_322[0]; + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_2 = r_280[0]; float gc_3 = gn_2 - 21.640425f; int eb_2 = 98304 + th * -32768; int ew_2 = eb_2 >> 2; @@ -5260,364 +4875,404 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int gb8_2 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_2 * 128); int kb8_2 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_2 * 128); float dbp_2[8]; - float qdg_2[8]; - tmem_ld_x8(&qdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float kdg_2[8]; - tmem_ld_x8(&kdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); + float qdg_0[8]; + tmem_ld_x8(&qdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); + float kdg_0[8]; + tmem_ld_x8(&kdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); float dkt_3[8]; tmem_ld_x8(&dkt_3[0], taddr + (unsigned int)row_base + (unsigned int)(8 * th)); - float btv_2[8]; + float btv_0[8]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_2[0])), "=r"(*reinterpret_cast(&btv_2[(0) + 1])), "=r"(*reinterpret_cast(&btv_2[(0) + 2])), "=r"(*reinterpret_cast(&btv_2[(0) + 3])) + : "=r"(*reinterpret_cast(&btv_0[0])), "=r"(*reinterpret_cast(&btv_0[(0) + 1])), "=r"(*reinterpret_cast(&btv_0[(0) + 2])), "=r"(*reinterpret_cast(&btv_0[(0) + 3])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_2[4])), "=r"(*reinterpret_cast(&btv_2[(4) + 1])), "=r"(*reinterpret_cast(&btv_2[(4) + 2])), "=r"(*reinterpret_cast(&btv_2[(4) + 3])) + : "=r"(*reinterpret_cast(&btv_0[4])), "=r"(*reinterpret_cast(&btv_0[(4) + 1])), "=r"(*reinterpret_cast(&btv_0[(4) + 2])), "=r"(*reinterpret_cast(&btv_0[(4) + 3])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th) + 16))); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_2[1]; s_run_2[0] = 0.0f; - int t_24 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_24 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_24 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_24 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_24 >= clen) ? 0.0f : q_t_24); - float dqf_24 = pool_f32[ew_2 + 896 + d_lane]; - float dkf_24 = pool_f32[ew_2 + 1920 + d_lane]; - float dgf_24 = pool_f32[ew_2 + 2944 + d_lane]; - float bt_t_24 = btv_2[7]; - float e_off_24 = approx_exp2(g_t_24 - gn_2); - float e_dg_24 = approx_exp2(g_t_24 - gc_3); - float dq2_24 = qdg_2[7] * e_dg_24; - float dk2_24 = kdg_2[7] * e_dg_24; - dbp_2[7] = dk2_24 * k_t_24; - float dk2b_24 = dk2_24 * bt_t_24; - float dq_o_24 = dq2_24 + dqf_24; - float dk_o_24 = dk2b_24 + dkf_24 + dkt_3[7]; - float dg_o_24 = __fmaf_rn(q_t_1_24, dq2_24, (dk2b_24 - dkt_3[7]) * k_t_24) + dgf_24; - s_run_2[0] = s_run_2[0] + dg_o_24; + int t_8 = 8 * th + 7; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_8 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_8 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_8 = __uint_as_float(r_283[0] << 16); + float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); + float bt_t_8 = btv_0[7]; + float e_off_8 = approx_exp2(g_t_8 - gn_2); + float e_dg_8 = approx_exp2(g_t_8 - gc_3); + float dq2_8 = qdg_0[7] * e_dg_8; + float dk2_8 = kdg_0[7] * e_dg_8; + dbp_2[7] = dk2_8 * k_t_8; + float dk2b_8 = dk2_8 * bt_t_8; + float qe_7 = q_t_1_8 * e_dg_8; + float kp_19 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_8) * kdg_0[7]); + float dqf_8 = pool_f32[ew_2 + 896 + d_lane]; + float dkf_8 = pool_f32[ew_2 + 1920 + d_lane]; + float dgf_8 = pool_f32[ew_2 + 2944 + d_lane]; + float dq_o_8 = dq2_8 + dqf_8; + float dk_o_8 = dk2b_8 + dkf_8 + dkt_3[7]; + float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_3[7], k_t_8, cr_7)) + dgf_8; + s_run_2[0] = s_run_2[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_24 = t_24 * qrs; - if (t_24 < clen) { + int qoff_8 = t_8 * qrs; + if (t_8 < clen) { { - auto _gst_487 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_487)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_487) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_491 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_491)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_488 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_488)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_488) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_492 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_492)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_8; } } - int t_25 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_25 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_25 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_25 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_25 >= clen) ? 0.0f : q_t_25); - float dqf_25 = pool_f32[ew_2 + 768 + d_lane]; - float dkf_25 = pool_f32[ew_2 + 1792 + d_lane]; - float dgf_25 = pool_f32[ew_2 + 2816 + d_lane]; - float bt_t_25 = btv_2[6]; - float e_off_25 = approx_exp2(g_t_25 - gn_2); - float e_dg_25 = approx_exp2(g_t_25 - gc_3); - float dq2_25 = qdg_2[6] * e_dg_25; - float dk2_25 = kdg_2[6] * e_dg_25; - dbp_2[6] = dk2_25 * k_t_25; - float dk2b_25 = dk2_25 * bt_t_25; - float dq_o_25 = dq2_25 + dqf_25; - float dk_o_25 = dk2b_25 + dkf_25 + dkt_3[6]; - float dg_o_25 = __fmaf_rn(q_t_1_25, dq2_25, (dk2b_25 - dkt_3[6]) * k_t_25) + dgf_25; - s_run_2[0] = s_run_2[0] + dg_o_25; + int t_9 = 8 * th + 6; + float r_284[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_9 = r_284[0]; + unsigned int r_285[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_9 = __uint_as_float(r_285[0] << 16); + unsigned int r_286[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_9 = __uint_as_float(r_286[0] << 16); + float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); + float bt_t_9 = btv_0[6]; + float e_off_9 = approx_exp2(g_t_9 - gn_2); + float e_dg_9 = approx_exp2(g_t_9 - gc_3); + float dq2_9 = qdg_0[6] * e_dg_9; + float dk2_9 = kdg_0[6] * e_dg_9; + dbp_2[6] = dk2_9 * k_t_9; + float dk2b_9 = dk2_9 * bt_t_9; + float qe_8 = q_t_1_9 * e_dg_9; + float kp_20 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_9) * kdg_0[6]); + float dqf_9 = pool_f32[ew_2 + 768 + d_lane]; + float dkf_9 = pool_f32[ew_2 + 1792 + d_lane]; + float dgf_9 = pool_f32[ew_2 + 2816 + d_lane]; + float dq_o_9 = dq2_9 + dqf_9; + float dk_o_9 = dk2b_9 + dkf_9 + dkt_3[6]; + float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_3[6], k_t_9, cr_8)) + dgf_9; + s_run_2[0] = s_run_2[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_25 = t_25 * qrs; - if (t_25 < clen) { + int qoff_9 = t_9 * qrs; + if (t_9 < clen) { { - auto _gst_489 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_489)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_489) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_493 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_493)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_490 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_490)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_490) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_494 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_494)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_9; } } - int t_26 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_26 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_26 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_26 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_26 >= clen) ? 0.0f : q_t_26); - float dqf_26 = pool_f32[ew_2 + 640 + d_lane]; - float dkf_26 = pool_f32[ew_2 + 1664 + d_lane]; - float dgf_26 = pool_f32[ew_2 + 2688 + d_lane]; - float bt_t_26 = btv_2[5]; - float e_off_26 = approx_exp2(g_t_26 - gn_2); - float e_dg_26 = approx_exp2(g_t_26 - gc_3); - float dq2_26 = qdg_2[5] * e_dg_26; - float dk2_26 = kdg_2[5] * e_dg_26; - dbp_2[5] = dk2_26 * k_t_26; - float dk2b_26 = dk2_26 * bt_t_26; - float dq_o_26 = dq2_26 + dqf_26; - float dk_o_26 = dk2b_26 + dkf_26 + dkt_3[5]; - float dg_o_26 = __fmaf_rn(q_t_1_26, dq2_26, (dk2b_26 - dkt_3[5]) * k_t_26) + dgf_26; - s_run_2[0] = s_run_2[0] + dg_o_26; + int t_10 = 8 * th + 5; + float r_287[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_10 = r_287[0]; + unsigned int r_288[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_10 = __uint_as_float(r_288[0] << 16); + unsigned int r_289[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_10 = __uint_as_float(r_289[0] << 16); + float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); + float bt_t_10 = btv_0[5]; + float e_off_10 = approx_exp2(g_t_10 - gn_2); + float e_dg_10 = approx_exp2(g_t_10 - gc_3); + float dq2_10 = qdg_0[5] * e_dg_10; + float dk2_10 = kdg_0[5] * e_dg_10; + dbp_2[5] = dk2_10 * k_t_10; + float dk2b_10 = dk2_10 * bt_t_10; + float qe_9 = q_t_1_10 * e_dg_10; + float kp_21 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_10) * kdg_0[5]); + float dqf_10 = pool_f32[ew_2 + 640 + d_lane]; + float dkf_10 = pool_f32[ew_2 + 1664 + d_lane]; + float dgf_10 = pool_f32[ew_2 + 2688 + d_lane]; + float dq_o_10 = dq2_10 + dqf_10; + float dk_o_10 = dk2b_10 + dkf_10 + dkt_3[5]; + float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_3[5], k_t_10, cr_9)) + dgf_10; + s_run_2[0] = s_run_2[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_26 = t_26 * qrs; - if (t_26 < clen) { + int qoff_10 = t_10 * qrs; + if (t_10 < clen) { { - auto _gst_491 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_491)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_495 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_495)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_492 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_492)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_496 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_496)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_10; } } - int t_27 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_27 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_27 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_27 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_27 >= clen) ? 0.0f : q_t_27); - float dqf_27 = pool_f32[ew_2 + 512 + d_lane]; - float dkf_27 = pool_f32[ew_2 + 1536 + d_lane]; - float dgf_27 = pool_f32[ew_2 + 2560 + d_lane]; - float bt_t_27 = btv_2[4]; - float e_off_27 = approx_exp2(g_t_27 - gn_2); - float e_dg_27 = approx_exp2(g_t_27 - gc_3); - float dq2_27 = qdg_2[4] * e_dg_27; - float dk2_27 = kdg_2[4] * e_dg_27; - dbp_2[4] = dk2_27 * k_t_27; - float dk2b_27 = dk2_27 * bt_t_27; - float dq_o_27 = dq2_27 + dqf_27; - float dk_o_27 = dk2b_27 + dkf_27 + dkt_3[4]; - float dg_o_27 = __fmaf_rn(q_t_1_27, dq2_27, (dk2b_27 - dkt_3[4]) * k_t_27) + dgf_27; - s_run_2[0] = s_run_2[0] + dg_o_27; + int t_11 = 8 * th + 4; + float r_290[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_11 = r_290[0]; + unsigned int r_291[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_11 = __uint_as_float(r_291[0] << 16); + unsigned int r_292[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_11 = __uint_as_float(r_292[0] << 16); + float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); + float bt_t_11 = btv_0[4]; + float e_off_11 = approx_exp2(g_t_11 - gn_2); + float e_dg_11 = approx_exp2(g_t_11 - gc_3); + float dq2_11 = qdg_0[4] * e_dg_11; + float dk2_11 = kdg_0[4] * e_dg_11; + dbp_2[4] = dk2_11 * k_t_11; + float dk2b_11 = dk2_11 * bt_t_11; + float qe_10 = q_t_1_11 * e_dg_11; + float kp_22 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_11) * kdg_0[4]); + float dqf_11 = pool_f32[ew_2 + 512 + d_lane]; + float dkf_11 = pool_f32[ew_2 + 1536 + d_lane]; + float dgf_11 = pool_f32[ew_2 + 2560 + d_lane]; + float dq_o_11 = dq2_11 + dqf_11; + float dk_o_11 = dk2b_11 + dkf_11 + dkt_3[4]; + float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_3[4], k_t_11, cr_10)) + dgf_11; + s_run_2[0] = s_run_2[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_27 = t_27 * qrs; - if (t_27 < clen) { + int qoff_11 = t_11 * qrs; + if (t_11 < clen) { { - auto _gst_493 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_493)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_497 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_497)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_494 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_494)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_498 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_498)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_11; } } - int t_28 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_28 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_28 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_28 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_28 >= clen) ? 0.0f : q_t_28); - float dqf_28 = pool_f32[ew_2 + 384 + d_lane]; - float dkf_28 = pool_f32[ew_2 + 1408 + d_lane]; - float dgf_28 = pool_f32[ew_2 + 2432 + d_lane]; - float bt_t_28 = btv_2[3]; - float e_off_28 = approx_exp2(g_t_28 - gn_2); - float e_dg_28 = approx_exp2(g_t_28 - gc_3); - float dq2_28 = qdg_2[3] * e_dg_28; - float dk2_28 = kdg_2[3] * e_dg_28; - dbp_2[3] = dk2_28 * k_t_28; - float dk2b_28 = dk2_28 * bt_t_28; - float dq_o_28 = dq2_28 + dqf_28; - float dk_o_28 = dk2b_28 + dkf_28 + dkt_3[3]; - float dg_o_28 = __fmaf_rn(q_t_1_28, dq2_28, (dk2b_28 - dkt_3[3]) * k_t_28) + dgf_28; - s_run_2[0] = s_run_2[0] + dg_o_28; + int t_12 = 8 * th + 3; + float r_293[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_12 = r_293[0]; + unsigned int r_294[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_12 = __uint_as_float(r_294[0] << 16); + unsigned int r_295[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_12 = __uint_as_float(r_295[0] << 16); + float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); + float bt_t_12 = btv_0[3]; + float e_off_12 = approx_exp2(g_t_12 - gn_2); + float e_dg_12 = approx_exp2(g_t_12 - gc_3); + float dq2_12 = qdg_0[3] * e_dg_12; + float dk2_12 = kdg_0[3] * e_dg_12; + dbp_2[3] = dk2_12 * k_t_12; + float dk2b_12 = dk2_12 * bt_t_12; + float qe_11 = q_t_1_12 * e_dg_12; + float kp_23 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_12) * kdg_0[3]); + float dqf_12 = pool_f32[ew_2 + 384 + d_lane]; + float dkf_12 = pool_f32[ew_2 + 1408 + d_lane]; + float dgf_12 = pool_f32[ew_2 + 2432 + d_lane]; + float dq_o_12 = dq2_12 + dqf_12; + float dk_o_12 = dk2b_12 + dkf_12 + dkt_3[3]; + float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_3[3], k_t_12, cr_11)) + dgf_12; + s_run_2[0] = s_run_2[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_28 = t_28 * qrs; - if (t_28 < clen) { + int qoff_12 = t_12 * qrs; + if (t_12 < clen) { { - auto _gst_495 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_495)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_499 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_499)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_496 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_496)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_500 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_500)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_12; } } - int t_29 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_29 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_29 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_29 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_29 >= clen) ? 0.0f : q_t_29); - float dqf_29 = pool_f32[ew_2 + 256 + d_lane]; - float dkf_29 = pool_f32[ew_2 + 1280 + d_lane]; - float dgf_29 = pool_f32[ew_2 + 2304 + d_lane]; - float bt_t_29 = btv_2[2]; - float e_off_29 = approx_exp2(g_t_29 - gn_2); - float e_dg_29 = approx_exp2(g_t_29 - gc_3); - float dq2_29 = qdg_2[2] * e_dg_29; - float dk2_29 = kdg_2[2] * e_dg_29; - dbp_2[2] = dk2_29 * k_t_29; - float dk2b_29 = dk2_29 * bt_t_29; - float dq_o_29 = dq2_29 + dqf_29; - float dk_o_29 = dk2b_29 + dkf_29 + dkt_3[2]; - float dg_o_29 = __fmaf_rn(q_t_1_29, dq2_29, (dk2b_29 - dkt_3[2]) * k_t_29) + dgf_29; - s_run_2[0] = s_run_2[0] + dg_o_29; + int t_13 = 8 * th + 2; + float r_296[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_13 = r_296[0]; + unsigned int r_297[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_13 = __uint_as_float(r_297[0] << 16); + unsigned int r_298[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_13 = __uint_as_float(r_298[0] << 16); + float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); + float bt_t_13 = btv_0[2]; + float e_off_13 = approx_exp2(g_t_13 - gn_2); + float e_dg_13 = approx_exp2(g_t_13 - gc_3); + float dq2_13 = qdg_0[2] * e_dg_13; + float dk2_13 = kdg_0[2] * e_dg_13; + dbp_2[2] = dk2_13 * k_t_13; + float dk2b_13 = dk2_13 * bt_t_13; + float qe_12 = q_t_1_13 * e_dg_13; + float kp_24 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_13) * kdg_0[2]); + float dqf_13 = pool_f32[ew_2 + 256 + d_lane]; + float dkf_13 = pool_f32[ew_2 + 1280 + d_lane]; + float dgf_13 = pool_f32[ew_2 + 2304 + d_lane]; + float dq_o_13 = dq2_13 + dqf_13; + float dk_o_13 = dk2b_13 + dkf_13 + dkt_3[2]; + float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_3[2], k_t_13, cr_12)) + dgf_13; + s_run_2[0] = s_run_2[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_29 = t_29 * qrs; - if (t_29 < clen) { + int qoff_13 = t_13 * qrs; + if (t_13 < clen) { { - auto _gst_497 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_497)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_501 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_501)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_498 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_498)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_502 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_502)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_13; } } - int t_30 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_30 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_30 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_30 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_30 >= clen) ? 0.0f : q_t_30); - float dqf_30 = pool_f32[ew_2 + 128 + d_lane]; - float dkf_30 = pool_f32[ew_2 + 1152 + d_lane]; - float dgf_30 = pool_f32[ew_2 + 2176 + d_lane]; - float bt_t_30 = btv_2[1]; - float e_off_30 = approx_exp2(g_t_30 - gn_2); - float e_dg_30 = approx_exp2(g_t_30 - gc_3); - float dq2_30 = qdg_2[1] * e_dg_30; - float dk2_30 = kdg_2[1] * e_dg_30; - dbp_2[1] = dk2_30 * k_t_30; - float dk2b_30 = dk2_30 * bt_t_30; - float dq_o_30 = dq2_30 + dqf_30; - float dk_o_30 = dk2b_30 + dkf_30 + dkt_3[1]; - float dg_o_30 = __fmaf_rn(q_t_1_30, dq2_30, (dk2b_30 - dkt_3[1]) * k_t_30) + dgf_30; - s_run_2[0] = s_run_2[0] + dg_o_30; + int t_14 = 8 * th + 1; + float r_299[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_14 = r_299[0]; + unsigned int r_300[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_14 = __uint_as_float(r_300[0] << 16); + unsigned int r_301[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_14 = __uint_as_float(r_301[0] << 16); + float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); + float bt_t_14 = btv_0[1]; + float e_off_14 = approx_exp2(g_t_14 - gn_2); + float e_dg_14 = approx_exp2(g_t_14 - gc_3); + float dq2_14 = qdg_0[1] * e_dg_14; + float dk2_14 = kdg_0[1] * e_dg_14; + dbp_2[1] = dk2_14 * k_t_14; + float dk2b_14 = dk2_14 * bt_t_14; + float qe_13 = q_t_1_14 * e_dg_14; + float kp_25 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_14) * kdg_0[1]); + float dqf_14 = pool_f32[ew_2 + 128 + d_lane]; + float dkf_14 = pool_f32[ew_2 + 1152 + d_lane]; + float dgf_14 = pool_f32[ew_2 + 2176 + d_lane]; + float dq_o_14 = dq2_14 + dqf_14; + float dk_o_14 = dk2b_14 + dkf_14 + dkt_3[1]; + float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_3[1], k_t_14, cr_13)) + dgf_14; + s_run_2[0] = s_run_2[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_30 = t_30 * qrs; - if (t_30 < clen) { + int qoff_14 = t_14 * qrs; + if (t_14 < clen) { { - auto _gst_499 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_499)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_503 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_503)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_503) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_500 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_500)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_504 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_504)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_504) + (0)) = (__nv_bfloat16)dk_o_14; } } - int t_31 = 0 + 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_31 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_31 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_31 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_31 >= clen) ? 0.0f : q_t_31); - float dqf_31 = pool_f32[ew_2 + d_lane]; - float dkf_31 = pool_f32[ew_2 + 1024 + d_lane]; - float dgf_31 = pool_f32[ew_2 + 2048 + d_lane]; - float bt_t_31 = btv_2[0]; - float e_off_31 = approx_exp2(g_t_31 - gn_2); - float e_dg_31 = approx_exp2(g_t_31 - gc_3); - float dq2_31 = qdg_2[0] * e_dg_31; - float dk2_31 = kdg_2[0] * e_dg_31; - dbp_2[0] = dk2_31 * k_t_31; - float dk2b_31 = dk2_31 * bt_t_31; - float dq_o_31 = dq2_31 + dqf_31; - float dk_o_31 = dk2b_31 + dkf_31 + dkt_3[0]; - float dg_o_31 = __fmaf_rn(q_t_1_31, dq2_31, (dk2b_31 - dkt_3[0]) * k_t_31) + dgf_31; - s_run_2[0] = s_run_2[0] + dg_o_31; + int t_15 = 0 + 8 * th; + float r_302[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_15 = r_302[0]; + unsigned int r_303[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_15 = __uint_as_float(r_303[0] << 16); + unsigned int r_304[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_15 = __uint_as_float(r_304[0] << 16); + float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); + float bt_t_15 = btv_0[0]; + float e_off_15 = approx_exp2(g_t_15 - gn_2); + float e_dg_15 = approx_exp2(g_t_15 - gc_3); + float dq2_15 = qdg_0[0] * e_dg_15; + float dk2_15 = kdg_0[0] * e_dg_15; + dbp_2[0] = dk2_15 * k_t_15; + float dk2b_15 = dk2_15 * bt_t_15; + float qe_14 = q_t_1_15 * e_dg_15; + float kp_26 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_15) * kdg_0[0]); + float dqf_15 = pool_f32[ew_2 + d_lane]; + float dkf_15 = pool_f32[ew_2 + 1024 + d_lane]; + float dgf_15 = pool_f32[ew_2 + 2048 + d_lane]; + float dq_o_15 = dq2_15 + dqf_15; + float dk_o_15 = dk2b_15 + dkf_15 + dkt_3[0]; + float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_3[0], k_t_15, cr_14)) + dgf_15; + s_run_2[0] = s_run_2[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_31 = t_31 * qrs; - if (t_31 < clen) { + int qoff_15 = t_15 * qrs; + if (t_15 < clen) { { - auto _gst_501 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_501)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_505 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_505)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_505) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_502 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_502)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_506 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_506)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_506) + (0)) = (__nv_bfloat16)dk_o_15; } } float tot_2 = s_run_2[0]; @@ -5626,37 +5281,37 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_503 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_503; + float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_507; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_504; + float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_508; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_505 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_505; + float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_509; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_506; + float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_510; float v2_18[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_18[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_507; + float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_18[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_511; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_18[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_508; + float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_18[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_512; float send_26 = ((b2_2 == 0) ? v2_18[1] : v2_18[0]); - float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_18[0] : v2_18[1]) + _shfl_xor_sync_509; - float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_510; - float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_511; + float _shfl_xor_sync_513 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_18[0] : v2_18[1]) + _shfl_xor_sync_513; + float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_514; + float _shfl_xor_sync_515 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_515; tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_512 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_512), "f"(tot_2) : "memory"); + uint32_t _addr_516 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_516), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5688,132 +5343,132 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_513 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_517 = make_float2(gacc[4], gacc[4]); float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_513), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_516; - asm("abs.f32 %0, %1;" : "=f"(_abs_516) : "f"(rp_z_11.x)); - float _neg_515; - asm("neg.f32 %0, %1;" : "=f"(_neg_515) : "f"(_abs_516)); - float _exp_514 = __nv_expf(_neg_515); - float _abs_519; - asm("abs.f32 %0, %1;" : "=f"(_abs_519) : "f"(rp_z_11.y)); - float _neg_518; - asm("neg.f32 %0, %1;" : "=f"(_neg_518) : "f"(_abs_519)); - float _exp_517 = __nv_expf(_neg_518); - float2 rp_e_11 = make_float2(_exp_514, _exp_517); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_517), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_520; + asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_11.x)); + float _neg_519; + asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); + float _exp_518 = __nv_expf(_neg_519); + float _abs_523; + asm("abs.f32 %0, %1;" : "=f"(_abs_523) : "f"(rp_z_11.y)); + float _neg_522; + asm("neg.f32 %0, %1;" : "=f"(_neg_522) : "f"(_abs_523)); + float _exp_521 = __nv_expf(_neg_522); + float2 rp_e_11 = make_float2(_exp_518, _exp_521); float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_520 = approx_rcp(rp_den_11.x); - float _rcp_521 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_522 = make_float2(_rcp_520, _rcp_521); + float _rcp_524 = approx_rcp(rp_den_11.x); + float _rcp_525 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_526 = make_float2(_rcp_524, _rcp_525); float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_522)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_526)); float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_523 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_527 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_527)); u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_524 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_528 = make_float2(gacc[4], gacc[4]); float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_524), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_527; - asm("abs.f32 %0, %1;" : "=f"(_abs_527) : "f"(rp_z_12.x)); - float _neg_526; - asm("neg.f32 %0, %1;" : "=f"(_neg_526) : "f"(_abs_527)); - float _exp_525 = __nv_expf(_neg_526); - float _abs_530; - asm("abs.f32 %0, %1;" : "=f"(_abs_530) : "f"(rp_z_12.y)); - float _neg_529; - asm("neg.f32 %0, %1;" : "=f"(_neg_529) : "f"(_abs_530)); - float _exp_528 = __nv_expf(_neg_529); - float2 rp_e_12 = make_float2(_exp_525, _exp_528); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_528), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_531; + asm("abs.f32 %0, %1;" : "=f"(_abs_531) : "f"(rp_z_12.x)); + float _neg_530; + asm("neg.f32 %0, %1;" : "=f"(_neg_530) : "f"(_abs_531)); + float _exp_529 = __nv_expf(_neg_530); + float _abs_534; + asm("abs.f32 %0, %1;" : "=f"(_abs_534) : "f"(rp_z_12.y)); + float _neg_533; + asm("neg.f32 %0, %1;" : "=f"(_neg_533) : "f"(_abs_534)); + float _exp_532 = __nv_expf(_neg_533); + float2 rp_e_12 = make_float2(_exp_529, _exp_532); float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_531 = approx_rcp(rp_den_12.x); - float _rcp_532 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_533 = make_float2(_rcp_531, _rcp_532); + float _rcp_535 = approx_rcp(rp_den_12.x); + float _rcp_536 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_537 = make_float2(_rcp_535, _rcp_536); float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_533)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_537)); float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_534 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_538 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_534)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_538)); u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_535 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_539 = make_float2(gacc[4], gacc[4]); float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_535), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_538; - asm("abs.f32 %0, %1;" : "=f"(_abs_538) : "f"(rp_z_13.x)); - float _neg_537; - asm("neg.f32 %0, %1;" : "=f"(_neg_537) : "f"(_abs_538)); - float _exp_536 = __nv_expf(_neg_537); - float _abs_541; - asm("abs.f32 %0, %1;" : "=f"(_abs_541) : "f"(rp_z_13.y)); - float _neg_540; - asm("neg.f32 %0, %1;" : "=f"(_neg_540) : "f"(_abs_541)); - float _exp_539 = __nv_expf(_neg_540); - float2 rp_e_13 = make_float2(_exp_536, _exp_539); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_539), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_542; + asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(rp_z_13.x)); + float _neg_541; + asm("neg.f32 %0, %1;" : "=f"(_neg_541) : "f"(_abs_542)); + float _exp_540 = __nv_expf(_neg_541); + float _abs_545; + asm("abs.f32 %0, %1;" : "=f"(_abs_545) : "f"(rp_z_13.y)); + float _neg_544; + asm("neg.f32 %0, %1;" : "=f"(_neg_544) : "f"(_abs_545)); + float _exp_543 = __nv_expf(_neg_544); + float2 rp_e_13 = make_float2(_exp_540, _exp_543); float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_542 = approx_rcp(rp_den_13.x); - float _rcp_543 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_544 = make_float2(_rcp_542, _rcp_543); + float _rcp_546 = approx_rcp(rp_den_13.x); + float _rcp_547 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_548 = make_float2(_rcp_546, _rcp_547); float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_544)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_548)); float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_545 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_549 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_545)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_549)); u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_546 = make_float2(gacc[4], gacc[4]); + float2 _f32x2_mul_a_550 = make_float2(gacc[4], gacc[4]); float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_546), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_549; - asm("abs.f32 %0, %1;" : "=f"(_abs_549) : "f"(rp_z_14.x)); - float _neg_548; - asm("neg.f32 %0, %1;" : "=f"(_neg_548) : "f"(_abs_549)); - float _exp_547 = __nv_expf(_neg_548); - float _abs_552; - asm("abs.f32 %0, %1;" : "=f"(_abs_552) : "f"(rp_z_14.y)); - float _neg_551; - asm("neg.f32 %0, %1;" : "=f"(_neg_551) : "f"(_abs_552)); - float _exp_550 = __nv_expf(_neg_551); - float2 rp_e_14 = make_float2(_exp_547, _exp_550); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_550), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_553; + asm("abs.f32 %0, %1;" : "=f"(_abs_553) : "f"(rp_z_14.x)); + float _neg_552; + asm("neg.f32 %0, %1;" : "=f"(_neg_552) : "f"(_abs_553)); + float _exp_551 = __nv_expf(_neg_552); + float _abs_556; + asm("abs.f32 %0, %1;" : "=f"(_abs_556) : "f"(rp_z_14.y)); + float _neg_555; + asm("neg.f32 %0, %1;" : "=f"(_neg_555) : "f"(_abs_556)); + float _exp_554 = __nv_expf(_neg_555); + float2 rp_e_14 = make_float2(_exp_551, _exp_554); float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_553 = approx_rcp(rp_den_14.x); - float _rcp_554 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_555 = make_float2(_rcp_553, _rcp_554); + float _rcp_557 = approx_rcp(rp_den_14.x); + float _rcp_558 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_559 = make_float2(_rcp_557, _rcp_558); float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_555)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_559)); float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_556 = make_float2(gacc[5], gacc[5]); + float2 _f32x2_mul_b_560 = make_float2(gacc[5], gacc[5]); float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_556)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_560)); u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -5828,54 +5483,54 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_557; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_557) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_557; + uint32_t _prmt_561; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_561; int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_558 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_558)); - *(reinterpret_cast(_gst_558) + (0)) = pq_12[0]; + auto _gst_562 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_562)); + *(reinterpret_cast(_gst_562) + (0)) = pq_12[0]; } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_559; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_559) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_559; + uint32_t _prmt_563; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_563; int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_560 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_560)); - *(reinterpret_cast(_gst_560) + (0)) = pq_13[0]; + auto _gst_564 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_564)); + *(reinterpret_cast(_gst_564) + (0)) = pq_13[0]; } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_561; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_561; + uint32_t _prmt_565; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_565) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_565; int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_562 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_562)); - *(reinterpret_cast(_gst_562) + (0)) = pq_14[0]; + auto _gst_566 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_566)); + *(reinterpret_cast(_gst_566) + (0)) = pq_14[0]; } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_563; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_563; + uint32_t _prmt_567; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_567) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_567; int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_564 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_564)); - *(reinterpret_cast(_gst_564) + (0)) = pq_15[0]; + auto _gst_568 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_568)); + *(reinterpret_cast(_gst_568) + (0)) = pq_15[0]; } } int t_a_24 = 8 * th; @@ -5940,47 +5595,47 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_565; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_565) : "f"(s_a_2), "f"(nm_t_5)); + float _sub_569; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_569) : "f"(s_a_2), "f"(nm_t_5)); float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_565), "f"(pa_2)); - float _sub_566; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_566) : "f"(pa_2), "f"(nm_t_5)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_569), "f"(pa_2)); + float _sub_570; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_570) : "f"(pa_2), "f"(nm_t_5)); float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_566), "f"(s_a_2)); - float _abs_568; - asm("abs.f32 %0, %1;" : "=f"(_abs_568) : "f"(s_a_2)); - float _abs_569; - asm("abs.f32 %0, %1;" : "=f"(_abs_569) : "f"(pa_2)); - uint32_t _setp_567_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_567_bits) : "f"(_abs_568), "f"(_abs_569)); - bool _setp_567 = _setp_567_bits != 0; - float nm_err_5 = ((_setp_567) ? nm_es_5 : nm_ex_5); - float _add_570; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_570; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_570), "f"(s_a_2)); + float _abs_572; + asm("abs.f32 %0, %1;" : "=f"(_abs_572) : "f"(s_a_2)); + float _abs_573; + asm("abs.f32 %0, %1;" : "=f"(_abs_573) : "f"(pa_2)); + uint32_t _setp_571_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_571_bits) : "f"(_abs_572), "f"(_abs_573)); + bool _setp_571 = _setp_571_bits != 0; + float nm_err_5 = ((_setp_571) ? nm_es_5 : nm_ex_5); + float _add_574; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_574; s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_571; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(s_b_2), "f"(nm_t_6)); + float _sub_575; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_575) : "f"(s_b_2), "f"(nm_t_6)); float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_571), "f"(pb_2)); - float _sub_572; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_572) : "f"(pb_2), "f"(nm_t_6)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_575), "f"(pb_2)); + float _sub_576; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_576) : "f"(pb_2), "f"(nm_t_6)); float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_572), "f"(s_b_2)); - float _abs_574; - asm("abs.f32 %0, %1;" : "=f"(_abs_574) : "f"(s_b_2)); - float _abs_575; - asm("abs.f32 %0, %1;" : "=f"(_abs_575) : "f"(pb_2)); - uint32_t _setp_573_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_573_bits) : "f"(_abs_574), "f"(_abs_575)); - bool _setp_573 = _setp_573_bits != 0; - float nm_err_6 = ((_setp_573) ? nm_es_6 : nm_ex_6); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_576; + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_576), "f"(s_b_2)); + float _abs_578; + asm("abs.f32 %0, %1;" : "=f"(_abs_578) : "f"(s_b_2)); + float _abs_579; + asm("abs.f32 %0, %1;" : "=f"(_abs_579) : "f"(pb_2)); + uint32_t _setp_577_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_577_bits) : "f"(_abs_578), "f"(_abs_579)); + bool _setp_577 = _setp_577_bits != 0; + float nm_err_6 = ((_setp_577) ? nm_es_6 : nm_ex_6); + float _add_580; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_580; s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -6019,60 +5674,60 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_578; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_578) : "f"(ts_s), "f"(ts_bb)); - float _sub_577; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_577) : "f"(s_a_3), "f"(_sub_578)); - float _sub_579; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_579) : "f"(other[0]), "f"(ts_bb)); + float _sub_582; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s), "f"(ts_bb)); + float _sub_581; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(s_a_3), "f"(_sub_582)); + float _sub_583; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(other[0]), "f"(ts_bb)); float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_577), "f"(_sub_579)); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_a_3), "f"(other[1])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_581), "f"(_sub_583)); + float _add_584; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(c_a_3), "f"(other[1])); float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_580), "f"(ts_e)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_584), "f"(ts_e)); float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_582; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_581; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(ts_s), "f"(_sub_582)); - float _sub_583; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(pm_c), "f"(ts_bb_0)); + float _sub_586; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_585; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s), "f"(_sub_586)); + float _sub_587; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_587) : "f"(pm_c), "f"(ts_bb_0)); float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_581), "f"(_sub_583)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_585), "f"(_sub_587)); s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_585; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_584; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_584) : "f"(s_b_3), "f"(_sub_585)); - float _sub_586; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(other[2]), "f"(ts_bb_1)); + float _sub_589; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_588; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(s_b_3), "f"(_sub_589)); + float _sub_590; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(other[2]), "f"(ts_bb_1)); float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_584), "f"(_sub_586)); - float _add_587; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_587) : "f"(c_b_3), "f"(other[3])); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_588), "f"(_sub_590)); + float _add_591; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_591) : "f"(c_b_3), "f"(other[3])); float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_587), "f"(ts_e_1)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_591), "f"(ts_e_1)); float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_589; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_588; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(ts_s_1), "f"(_sub_589)); - float _sub_590; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(pm_c_0), "f"(ts_bb_2)); + float _sub_593; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_593) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_592; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_592) : "f"(ts_s_1), "f"(_sub_593)); + float _sub_594; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_594) : "f"(pm_c_0), "f"(ts_bb_2)); float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_588), "f"(_sub_590)); + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_592), "f"(_sub_594)); s_b_3 = ts_s_2; c_b_3 = ts_e_2; double v_a = (double)s_a_3 + (double)c_a_3; @@ -6175,41 +5830,41 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + for (int r_305 = 0; r_305 < 16; r_305++) { + int t_16 = r_305; + if (t_16 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6276,41 +5931,41 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_32 = 16 + r_348; - if (t_32 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + for (int r_306 = 0; r_306 < 16; r_306++) { + int t_17 = 16 + r_306; + if (t_17 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6416,41 +6071,41 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_33 = 32 + r_349; - if (t_33 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + for (int r_307 = 0; r_307 < 16; r_307++) { + int t_18 = 32 + r_307; + if (t_18 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6583,41 +6238,41 @@ kernel_cake_kda_train_31fa936f5468b5749a33(const __grid_constant__ CUtensorMap d z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_34 = 48 + r_350; - if (t_34 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + for (int r_308 = 0; r_308 < 16; r_308++) { + int t_19 = 48 + r_308; + if (t_19 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); } } asm volatile("tcgen05.fence::before_thread_sync;"); diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu index acde3620584..1e8f10c85cc 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_87b73f813c841159bf43_kernel.cu @@ -113,6 +113,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_ } + __device__ __forceinline__ void tmem_ld_x4(float* dst, int tmem_addr) { asm volatile( "tcgen05.ld.sync.aligned.32x32b.x4.b32" @@ -199,6 +200,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2267,12 +2284,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_1_0 = sub_f32x2(_f2_56, _f2_4); float en_0 = approx_exp2(dn2_1_0.x); float en_2_0 = approx_exp2(dn2_1_0.y); - dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * e2_40); - dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * e2_2_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_1_0, k_t1_1_0); float2 _f32x2_mul_b_169 = make_float2(e2_40, e2_2_0); float2 p2_1_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_1_0.x) & 4294959104u) - p2_1_0.x; + float _rcp_170 = approx_rcp(k_t0_1_0); + float ft = ((dl == 0.0f) ? e2_40 : __fmaf_rn(dl, _rcp_170, e2_40)); + float dl_0 = __uint_as_float(__as_u32(p2_1_0.y) & 4294959104u) - p2_1_0.y; + float _rcp_171 = approx_rcp(k_t1_1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_2_0 : __fmaf_rn(dl_0, _rcp_171, e2_2_0)); + dkt[0] = __fmaf_rn(_tmem_load_1[0], en_0, _tmem_load_0[0] * ft); + dkt[1] = __fmaf_rn(_tmem_load_1[1], en_2_0, _tmem_load_0[1] * ft_0); pan[0] = p2_1_0.x; pan[1] = p2_1_0.y; int t0_1_1 = 32 + 8 * th + 2; @@ -2296,12 +2319,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_1_1 = sub_f32x2(_f2_56, _f2_5); float en_1 = approx_exp2(dn2_1_1.x); float en_2_1 = approx_exp2(dn2_1_1.y); - dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * e2_41); - dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * e2_2_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1_1, k_t1_1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_41, e2_2_1); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1_1, k_t1_1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_41, e2_2_1); float2 p2_1_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1_1.x) & 4294959104u) - p2_1_1.x; + float _rcp_174 = approx_rcp(k_t0_1_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_41 : __fmaf_rn(dl_1, _rcp_174, e2_41)); + float dl_2 = __uint_as_float(__as_u32(p2_1_1.y) & 4294959104u) - p2_1_1.y; + float _rcp_175 = approx_rcp(k_t1_1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_2_1 : __fmaf_rn(dl_2, _rcp_175, e2_2_1)); + dkt[2] = __fmaf_rn(_tmem_load_1[2], en_1, _tmem_load_0[2] * ft_1); + dkt[3] = __fmaf_rn(_tmem_load_1[3], en_2_1, _tmem_load_0[3] * ft_2); pan[2] = p2_1_1.x; pan[3] = p2_1_1.y; int t0_1_2 = 32 + 8 * th + 4; @@ -2325,12 +2354,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_1_2 = sub_f32x2(_f2_56, _f2_6); float en_2 = approx_exp2(dn2_1_2.x); float en_2_2 = approx_exp2(dn2_1_2.y); - dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * e2_42); - dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * e2_2_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_1_2, k_t1_1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_42, e2_2_2); + float2 _f32x2_mul_a_176 = make_float2(k_t0_1_2, k_t1_1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_42, e2_2_2); float2 p2_1_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_1_2.x) & 4294959104u) - p2_1_2.x; + float _rcp_178 = approx_rcp(k_t0_1_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_42 : __fmaf_rn(dl_3, _rcp_178, e2_42)); + float dl_4 = __uint_as_float(__as_u32(p2_1_2.y) & 4294959104u) - p2_1_2.y; + float _rcp_179 = approx_rcp(k_t1_1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_2_2 : __fmaf_rn(dl_4, _rcp_179, e2_2_2)); + dkt[4] = __fmaf_rn(_tmem_load_1[4], en_2, _tmem_load_0[4] * ft_3); + dkt[5] = __fmaf_rn(_tmem_load_1[5], en_2_2, _tmem_load_0[5] * ft_4); pan[4] = p2_1_2.x; pan[5] = p2_1_2.y; int t0_1_3 = 32 + 8 * th + 6; @@ -2354,12 +2389,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_1_3 = sub_f32x2(_f2_56, _f2_7); float en_3 = approx_exp2(dn2_1_3.x); float en_2_3 = approx_exp2(dn2_1_3.y); - dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * e2_43); - dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * e2_2_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_1_3, k_t1_1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_43, e2_2_3); + float2 _f32x2_mul_a_180 = make_float2(k_t0_1_3, k_t1_1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_43, e2_2_3); float2 p2_1_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_1_3.x) & 4294959104u) - p2_1_3.x; + float _rcp_182 = approx_rcp(k_t0_1_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_43 : __fmaf_rn(dl_5, _rcp_182, e2_43)); + float dl_6 = __uint_as_float(__as_u32(p2_1_3.y) & 4294959104u) - p2_1_3.y; + float _rcp_183 = approx_rcp(k_t1_1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_2_3 : __fmaf_rn(dl_6, _rcp_183, e2_2_3)); + dkt[6] = __fmaf_rn(_tmem_load_1[6], en_3, _tmem_load_0[6] * ft_5); + dkt[7] = __fmaf_rn(_tmem_load_1[7], en_2_3, _tmem_load_0[7] * ft_6); pan[6] = p2_1_3.x; pan[7] = p2_1_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2395,12 +2436,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(_f2_61, make_float2(g_t0_0, g_t1_0)); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - dkt_640[0] = _tmem_load_2[0] * e2_0_0; - dkt_640[1] = _tmem_load_2[1] * e2_1_0; - float2 _f32x2_mul_a_176 = make_float2(k_t0_0, k_t1_0); - float2 _f32x2_mul_b_177 = make_float2(e2_0_0, e2_1_0); + float2 _f32x2_mul_a_184 = make_float2(k_t0_0, k_t1_0); + float2 _f32x2_mul_b_185 = make_float2(e2_0_0, e2_1_0); float2 p2_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_186 = approx_rcp(k_t0_0); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_0 : __fmaf_rn(dl_7, _rcp_186, e2_0_0)); + float dl_8 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_187 = approx_rcp(k_t1_0); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_8, _rcp_187, e2_1_0)); + dkt_640[0] = _tmem_load_2[0] * ft_7; + dkt_640[1] = _tmem_load_2[1] * ft_8; pan_641[0] = p2_0.x; pan_641[1] = p2_0.y; int t0_1 = 48 + 8 * th + 2; @@ -2420,12 +2467,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(_f2_61, make_float2(g_t0_1, g_t1_1)); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); - dkt_640[2] = _tmem_load_2[2] * e2_0_1; - dkt_640[3] = _tmem_load_2[3] * e2_1_1; - float2 _f32x2_mul_a_178 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_179 = make_float2(e2_0_1, e2_1_1); + float2 _f32x2_mul_a_188 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_189 = make_float2(e2_0_1, e2_1_1); float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_190 = approx_rcp(k_t0_1); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_9, _rcp_190, e2_0_1)); + float dl_10 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_191 = approx_rcp(k_t1_1); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_10, _rcp_191, e2_1_1)); + dkt_640[2] = _tmem_load_2[2] * ft_9; + dkt_640[3] = _tmem_load_2[3] * ft_10; pan_641[2] = p2_1.x; pan_641[3] = p2_1.y; int t0_2 = 48 + 8 * th + 4; @@ -2445,12 +2498,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(_f2_61, make_float2(g_t0_2, g_t1_2)); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); - dkt_640[4] = _tmem_load_2[4] * e2_0_2; - dkt_640[5] = _tmem_load_2[5] * e2_1_2; - float2 _f32x2_mul_a_180 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_181 = make_float2(e2_0_2, e2_1_2); + float2 _f32x2_mul_a_192 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_193 = make_float2(e2_0_2, e2_1_2); float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_194 = approx_rcp(k_t0_2); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_11, _rcp_194, e2_0_2)); + float dl_12 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_195 = approx_rcp(k_t1_2); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_12, _rcp_195, e2_1_2)); + dkt_640[4] = _tmem_load_2[4] * ft_11; + dkt_640[5] = _tmem_load_2[5] * ft_12; pan_641[4] = p2_2.x; pan_641[5] = p2_2.y; int t0_3 = 48 + 8 * th + 6; @@ -2470,12 +2529,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(_f2_61, make_float2(g_t0_3, g_t1_3)); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); - dkt_640[6] = _tmem_load_2[6] * e2_0_3; - dkt_640[7] = _tmem_load_2[7] * e2_1_3; - float2 _f32x2_mul_a_182 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_183 = make_float2(e2_0_3, e2_1_3); + float2 _f32x2_mul_a_196 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_197 = make_float2(e2_0_3, e2_1_3); float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_198 = approx_rcp(k_t0_3); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_13, _rcp_198, e2_0_3)); + float dl_14 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_199 = approx_rcp(k_t1_3); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_14, _rcp_199, e2_1_3)); + dkt_640[6] = _tmem_load_2[6] * ft_13; + dkt_640[7] = _tmem_load_2[7] * ft_14; pan_641[6] = p2_3.x; pan_641[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_640); @@ -2523,12 +2588,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_0 = sub_f32x2(_f2_64, _f2_8); float en_0_0 = approx_exp2(dn2_0.x); float en_1_0 = approx_exp2(dn2_0.y); - dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * e2_0_4); - dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * e2_1_4); - float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_200 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_201 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_202 = approx_rcp(k_t0_4); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_15, _rcp_202, e2_0_4)); + float dl_16 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_203 = approx_rcp(k_t1_4); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_16, _rcp_203, e2_1_4)); + dkt_702[0] = __fmaf_rn(_tmem_load_4[0], en_0_0, _tmem_load_3[0] * ft_15); + dkt_702[1] = __fmaf_rn(_tmem_load_4[1], en_1_0, _tmem_load_3[1] * ft_16); pan_703[0] = p2_4.x; pan_703[1] = p2_4.y; int t0_5 = 8 * th + 2; @@ -2552,12 +2623,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_1 = sub_f32x2(_f2_64, _f2_9); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * e2_0_5); - dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * e2_1_5); - float2 _f32x2_mul_a_186 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_187 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_204 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_205 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_206 = approx_rcp(k_t0_5); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_17, _rcp_206, e2_0_5)); + float dl_18 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_207 = approx_rcp(k_t1_5); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_18, _rcp_207, e2_1_5)); + dkt_702[2] = __fmaf_rn(_tmem_load_4[2], en_0_1, _tmem_load_3[2] * ft_17); + dkt_702[3] = __fmaf_rn(_tmem_load_4[3], en_1_1, _tmem_load_3[3] * ft_18); pan_703[2] = p2_5.x; pan_703[3] = p2_5.y; int t0_6 = 8 * th + 4; @@ -2581,12 +2658,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_2 = sub_f32x2(_f2_64, _f2_10); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * e2_0_6); - dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * e2_1_6); - float2 _f32x2_mul_a_188 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_189 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_208 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_209 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_210 = approx_rcp(k_t0_6); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_19, _rcp_210, e2_0_6)); + float dl_20 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_211 = approx_rcp(k_t1_6); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_20, _rcp_211, e2_1_6)); + dkt_702[4] = __fmaf_rn(_tmem_load_4[4], en_0_2, _tmem_load_3[4] * ft_19); + dkt_702[5] = __fmaf_rn(_tmem_load_4[5], en_1_2, _tmem_load_3[5] * ft_20); pan_703[4] = p2_6.x; pan_703[5] = p2_6.y; int t0_7 = 8 * th + 6; @@ -2610,12 +2693,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_3 = sub_f32x2(_f2_64, _f2_11); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * e2_0_7); - dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * e2_1_7); - float2 _f32x2_mul_a_190 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_191 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_212 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_213 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_214 = approx_rcp(k_t0_7); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_21, _rcp_214, e2_0_7)); + float dl_22 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_215 = approx_rcp(k_t1_7); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_22, _rcp_215, e2_1_7)); + dkt_702[6] = __fmaf_rn(_tmem_load_4[6], en_0_3, _tmem_load_3[6] * ft_21); + dkt_702[7] = __fmaf_rn(_tmem_load_4[7], en_1_3, _tmem_load_3[7] * ft_22); pan_703[6] = p2_7.x; pan_703[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_702); @@ -2657,12 +2746,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_4 = sub_f32x2(_f2_70, _f2_12); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * e2_0_8); - dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * e2_1_8); - float2 _f32x2_mul_a_192 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_193 = make_float2(e2_0_8, e2_1_8); + float2 _f32x2_mul_a_216 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_217 = make_float2(e2_0_8, e2_1_8); float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_218 = approx_rcp(k_t0_8); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_23, _rcp_218, e2_0_8)); + float dl_24 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_219 = approx_rcp(k_t1_8); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_24, _rcp_219, e2_1_8)); + dkt_776[0] = __fmaf_rn(_tmem_load_6[0], en_0_4, _tmem_load_5[0] * ft_23); + dkt_776[1] = __fmaf_rn(_tmem_load_6[1], en_1_4, _tmem_load_5[1] * ft_24); pan_777[0] = p2_8.x; pan_777[1] = p2_8.y; int t0_9 = 16 + 8 * th + 2; @@ -2686,12 +2781,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_5 = sub_f32x2(_f2_70, _f2_13); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * e2_0_9); - dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * e2_1_9); - float2 _f32x2_mul_a_194 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_195 = make_float2(e2_0_9, e2_1_9); + float2 _f32x2_mul_a_220 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_221 = make_float2(e2_0_9, e2_1_9); float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_222 = approx_rcp(k_t0_9); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_25, _rcp_222, e2_0_9)); + float dl_26 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_223 = approx_rcp(k_t1_9); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_26, _rcp_223, e2_1_9)); + dkt_776[2] = __fmaf_rn(_tmem_load_6[2], en_0_5, _tmem_load_5[2] * ft_25); + dkt_776[3] = __fmaf_rn(_tmem_load_6[3], en_1_5, _tmem_load_5[3] * ft_26); pan_777[2] = p2_9.x; pan_777[3] = p2_9.y; int t0_10 = 16 + 8 * th + 4; @@ -2715,12 +2816,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_6 = sub_f32x2(_f2_70, _f2_14); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * e2_0_10); - dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * e2_1_10); - float2 _f32x2_mul_a_196 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_197 = make_float2(e2_0_10, e2_1_10); + float2 _f32x2_mul_a_224 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_225 = make_float2(e2_0_10, e2_1_10); float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_226 = approx_rcp(k_t0_10); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_27, _rcp_226, e2_0_10)); + float dl_28 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_227 = approx_rcp(k_t1_10); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_28, _rcp_227, e2_1_10)); + dkt_776[4] = __fmaf_rn(_tmem_load_6[4], en_0_6, _tmem_load_5[4] * ft_27); + dkt_776[5] = __fmaf_rn(_tmem_load_6[5], en_1_6, _tmem_load_5[5] * ft_28); pan_777[4] = p2_10.x; pan_777[5] = p2_10.y; int t0_11 = 16 + 8 * th + 6; @@ -2744,12 +2851,18 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 dn2_7 = sub_f32x2(_f2_70, _f2_15); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * e2_0_11); - dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * e2_1_11); - float2 _f32x2_mul_a_198 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_199 = make_float2(e2_0_11, e2_1_11); + float2 _f32x2_mul_a_228 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_229 = make_float2(e2_0_11, e2_1_11); float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_230 = approx_rcp(k_t0_11); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_29, _rcp_230, e2_0_11)); + float dl_30 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_231 = approx_rcp(k_t1_11); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_30, _rcp_231, e2_1_11)); + dkt_776[6] = __fmaf_rn(_tmem_load_6[6], en_0_7, _tmem_load_5[6] * ft_29); + dkt_776[7] = __fmaf_rn(_tmem_load_6[7], en_1_7, _tmem_load_5[7] * ft_30); pan_777[6] = p2_11.x; pan_777[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_776); @@ -2820,37 +2933,37 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[0], gs_544_0[1])); float e0_12 = approx_exp2(d2_60.x); float e1_12 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_545_0[0], ks_545_0[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_12, e1_12); + float2 _f32x2_mul_a_232 = make_float2(ks_545_0[0], ks_545_0[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_12, e1_12); float2 v2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_562_0[0] = v2_12.x; v_562_0[1] = v2_12.y; float2 d2_61 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[2], gs_544_0[3])); float e0_13 = approx_exp2(d2_61.x); float e1_13 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_545_0[2], ks_545_0[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_13, e1_13); + float2 _f32x2_mul_a_234 = make_float2(ks_545_0[2], ks_545_0[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_13, e1_13); float2 v2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_562_0[2] = v2_13.x; v_562_0[3] = v2_13.y; float2 d2_62 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[4], gs_544_0[5])); float e0_14 = approx_exp2(d2_62.x); float e1_14 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_545_0[4], ks_545_0[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_14, e1_14); + float2 _f32x2_mul_a_236 = make_float2(ks_545_0[4], ks_545_0[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_14, e1_14); float2 v2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_562_0[4] = v2_14.x; v_562_0[5] = v2_14.y; float2 d2_63 = sub_f32x2(_f2_54_0, make_float2(gs_544_0[6], gs_544_0[7])); float e0_15 = approx_exp2(d2_63.x); float e1_15 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_545_0[6], ks_545_0[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_15, e1_15); + float2 _f32x2_mul_a_238 = make_float2(ks_545_0[6], ks_545_0[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_15, e1_15); float2 v2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_562_0[6] = v2_15.x; v_562_0[7] = v2_15.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_562_0[0]), "f"(v_562_0[1]), "f"(v_562_0[2]), "f"(v_562_0[3]) : "memory"); @@ -2862,29 +2975,29 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int kk_3_0 = w * 4; if (lane_1 > (kk_3_0 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_0 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_3_1 = w * 4 + 1; if (lane_1 > (kk_3_1 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_1 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_3_2 = w * 4 + 2; if (lane_1 > (kk_3_2 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_2 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_3_3 = w * 4 + 3; if (lane_1 > (kk_3_3 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_3_3 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2892,29 +3005,29 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int kk_3_4 = w * 4; if (lane_1 > (kk_3_4 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_4 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_3_5 = w * 4 + 1; if (lane_1 > (kk_3_5 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_5 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_3_6 = w * 4 + 2; if (lane_1 > (kk_3_6 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_6 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_3_7 = w * 4 + 3; if (lane_1 > (kk_3_7 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_3_7 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2940,885 +3053,801 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int sb8 = 32 + 8 * th; int gb8 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8 * 128); int kb8 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8 * 128); + int c_bp = 144 + 8 * th; + float s_run[1]; + s_run[0] = 0.0f; + #pragma unroll 1 + for (int k = 0; k < 8; k++) { + int j = 7 - k; + float qo[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qo[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + j))); + float ko[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(ko[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th + j))); + float qdg[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qdg[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + j))); + float kdg[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(kdg[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th + j))); + float dkt_0[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(dkt_0[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th + j))); + float btv[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btv[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th + j)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t = sb8 + j; + float r_248[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t = r_248[0]; + unsigned int r_249[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t = __uint_as_float(r_249[0] << 16); + unsigned int r_250[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t = __uint_as_float(r_250[0] << 16); + float q_t_1 = ((t >= clen) ? 0.0f : q_t); + float bt_t = btv[0]; + float e_off = approx_exp2(g_t - gn); + float e_dg = approx_exp2(g_t - gc_0); + float dq2 = __fmaf_rn(qo[0], e_off, qdg[0] * e_dg); + float dk2 = __fmaf_rn(ko[0], e_off, kdg[0] * e_dg); + float dbv[1]; + dbv[0] = dk2 * k_t; + float dk2b = dk2 * bt_t; + float qe = q_t_1 * e_dg; + float kp = k_t * bt_t; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg) * kdg[0]); + float dqf = pool_f32[ew + 128 * j + d_lane]; + float dkf = pool_f32[ew + (1024 + 128 * j) + d_lane]; + float dgf = pool_f32[ew + (2048 + 128 * j) + d_lane]; + float dq_o = dq2 + dqf; + float dk_o = dk2b + dkf + dkt_0[0]; + float dg_o = __fmaf_rn(q_t_1, dq2, __fmaf_rn(dk2b - dkt_0[0], k_t, cr)) + dgf; + s_run[0] = s_run[0] + dg_o; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + j)), "f"(s_run[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp + j)), "f"(dbv[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff = t * qrs; + if (t < clen) { + { + auto _gst_248 = dq_base + qoff; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o; + } + { + auto _gst_249 = dk_base + qoff; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o; + } + } + } + float tot = s_run[0]; float dbp[8]; - float qo[8]; - tmem_ld_x8(&qo[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float ko[8]; - tmem_ld_x8(&ko[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); - float qdg[8]; - tmem_ld_x8(&qdg[0], taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)); - float kdg[8]; - tmem_ld_x8(&kdg[0], taddr + (unsigned int)row_base + (unsigned int)(208 + 8 * th)); - float dkt_0[8]; - tmem_ld_x8(&dkt_0[0], taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th)); - float btv[8]; + tmem_ld_x8(&dbp[0], taddr + (unsigned int)row_base + (unsigned int)c_bp); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int b4 = lane_1 >> 4 & 1; + int b3 = lane_1 >> 3 & 1; + int b2 = lane_1 >> 2 & 1; + float v4[4]; + float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); + float _shfl_xor_sync_250 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_250; + float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); + float _shfl_xor_sync_251 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_251; + float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); + float _shfl_xor_sync_252 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_252; + float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); + float _shfl_xor_sync_253 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_253; + float v2[2]; + float send_4 = ((b3 == 0) ? v4[2] : v4[0]); + float _shfl_xor_sync_254 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_254; + float send_5 = ((b3 == 0) ? v4[3] : v4[1]); + float _shfl_xor_sync_255 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_255; + float send = ((b2 == 0) ? v2[1] : v2[0]); + float _shfl_xor_sync_256 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_256; + float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_257; + float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_258; + asm volatile("tcgen05.fence::before_thread_sync;"); + mbarrier_arrive(p1a_t2_drained_addr); + mbarrier_arrive(epi_empty_addr + (th) * 8); + int pf_row = 32 + 8 * th + (lane_1 & 7) + 16 * (lane_1 >> 3); + long long pf_off = (ext0 + (long long)pf_row) * g_raw_row_stride + (long long)hv * g_raw_head_stride; + if (lane_1 < 16 && pf_row < clen) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(g_raw + (pf_off + (long long)(32 * qd))))); } + if (lane_1 == 16) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(A_log + hv))); } + if (lane_1 == 17) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(dt_bias + (hv * 128 + 32 * qd)))); } + mbarrier_wait(epi_full_addr + (th) * 8, 1); + asm volatile("tcgen05.fence::after_thread_sync;"); + float r_251[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_0 = r_251[0]; + float gc_1 = gn_0 - 21.640425f; + int eb_0 = 98304 + th * -32768; + int ew_0 = eb_0 >> 2; + int qbase_1 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); + int sb8_0 = 48 + 8 * th; + int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); + int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); +#if __CUDA_ARCH__ == 1030 + float dbp_0[8]; + float qo_1[8]; + tmem_ld_x8(&qo_1[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); + float ko_1[8]; + tmem_ld_x8(&ko_1[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); + float qdg_1[8]; + tmem_ld_x8(&qdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); + float kdg_1[8]; + tmem_ld_x8(&kdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); + float dkt_1[8]; + tmem_ld_x8(&dkt_1[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); + float btv_1[8]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[0])), "=r"(*reinterpret_cast(&btv[(0) + 1])), "=r"(*reinterpret_cast(&btv[(0) + 2])), "=r"(*reinterpret_cast(&btv[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th)))); + : "=r"(*reinterpret_cast(&btv_1[0])), "=r"(*reinterpret_cast(&btv_1[(0) + 1])), "=r"(*reinterpret_cast(&btv_1[(0) + 2])), "=r"(*reinterpret_cast(&btv_1[(0) + 3])) + : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv[4])), "=r"(*reinterpret_cast(&btv[(4) + 1])), "=r"(*reinterpret_cast(&btv[(4) + 2])), "=r"(*reinterpret_cast(&btv[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (32 + 8 * th) + 16))); + : "=r"(*reinterpret_cast(&btv_1[4])), "=r"(*reinterpret_cast(&btv_1[(4) + 1])), "=r"(*reinterpret_cast(&btv_1[(4) + 2])), "=r"(*reinterpret_cast(&btv_1[(4) + 3])) + : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run[1]; - s_run[0] = 0.0f; - int t_0 = 32 + 8 * th + 7; - float r_248[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_248[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_0 = r_248[0]; - unsigned int r_249[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_249[0])) : "r"((kb8 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_0 = __uint_as_float(r_249[0] << 16); - unsigned int r_250[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_0 = __uint_as_float(r_250[0] << 16); +#else + int c_bp_0 = 176 + 8 * th; +#endif + float s_run_0[1]; + s_run_0[0] = 0.0f; +#if __CUDA_ARCH__ == 1030 + int t_0 = 48 + 8 * th + 7; + float r_252[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_0 = r_252[0]; + unsigned int r_253[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_0 = __uint_as_float(r_253[0] << 16); + unsigned int r_254[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_0 = __uint_as_float(r_254[0] << 16); float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew + 896 + d_lane]; - float dkf_0 = pool_f32[ew + 1920 + d_lane]; - float dgf_0 = pool_f32[ew + 2944 + d_lane]; - float bt_t_0 = btv[7]; - float e_off_0 = approx_exp2(g_t_0 - gn); - float e_dg_0 = approx_exp2(g_t_0 - gc_0); - float dq2_0 = __fmaf_rn(qo[7], e_off_0, qdg[7] * e_dg_0); - float dk2_0 = __fmaf_rn(ko[7], e_off_0, kdg[7] * e_dg_0); - dbp[7] = dk2_0 * k_t_0; + float bt_t_0 = btv_1[7]; + float e_off_0 = approx_exp2(g_t_0 - gn_0); + float e_dg_0 = approx_exp2(g_t_0 - gc_1); + float dq2_0 = __fmaf_rn(qo_1[7], e_off_0, qdg_1[7] * e_dg_0); + float dk2_0 = __fmaf_rn(ko_1[7], e_off_0, kdg_1[7] * e_dg_0); + dbp_0[7] = dk2_0 * k_t_0; float dk2b_0 = dk2_0 * bt_t_0; + float qe_1 = q_t_1_0 * e_dg_0; + float kp_12 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_12; + float kbe_1 = (float)kb_1 * e_dg_0; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg_1[7], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_12 * e_dg_0) * kdg_1[7]); + float dqf_0 = pool_f32[ew_0 + 896 + d_lane]; + float dkf_0 = pool_f32[ew_0 + 1920 + d_lane]; + float dgf_0 = pool_f32[ew_0 + 2944 + d_lane]; float dq_o_0 = dq2_0 + dqf_0; - float dk_o_0 = dk2b_0 + dkf_0 + dkt_0[7]; - float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, (dk2b_0 - dkt_0[7]) * k_t_0) + dgf_0; - s_run[0] = s_run[0] + dg_o_0; + float dk_o_0 = dk2b_0 + dkf_0 + dkt_1[7]; + float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_1[7], k_t_0, cr_1)) + dgf_0; + s_run_0[0] = s_run_0[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 7)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_259 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_260 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_0; } } - int t_1 = 32 + 8 * th + 6; - float r_251[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_251[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_1 = r_251[0]; - unsigned int r_252[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((kb8 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_1 = __uint_as_float(r_252[0] << 16); - unsigned int r_253[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_1 = __uint_as_float(r_253[0] << 16); - float q_t_1_1 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew + 768 + d_lane]; - float dkf_1 = pool_f32[ew + 1792 + d_lane]; - float dgf_1 = pool_f32[ew + 2816 + d_lane]; - float bt_t_1 = btv[6]; - float e_off_1 = approx_exp2(g_t_1 - gn); - float e_dg_1 = approx_exp2(g_t_1 - gc_0); - float dq2_1 = __fmaf_rn(qo[6], e_off_1, qdg[6] * e_dg_1); - float dk2_1 = __fmaf_rn(ko[6], e_off_1, kdg[6] * e_dg_1); - dbp[6] = dk2_1 * k_t_1; + int t_1 = 48 + 8 * th + 6; + float r_255[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_255[0]; + unsigned int r_256[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_256[0] << 16); + unsigned int r_257[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1_1 = __uint_as_float(r_257[0] << 16); + float q_t_1_1_1 = ((t_1 >= clen) ? 0.0f : q_t_1_1); + float bt_t_1 = btv_1[6]; + float e_off_1 = approx_exp2(g_t_1 - gn_0); + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float dq2_1 = __fmaf_rn(qo_1[6], e_off_1, qdg_1[6] * e_dg_1); + float dk2_1 = __fmaf_rn(ko_1[6], e_off_1, kdg_1[6] * e_dg_1); + dbp_0[6] = dk2_1 * k_t_1; float dk2b_1 = dk2_1 * bt_t_1; + float qe_0 = q_t_1_1_1 * e_dg_1; + float kp_12_1 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12_1; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg_1[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12_1 * e_dg_1) * kdg_1[6]); + float dqf_1 = pool_f32[ew_0 + 768 + d_lane]; + float dkf_1 = pool_f32[ew_0 + 1792 + d_lane]; + float dgf_1 = pool_f32[ew_0 + 2816 + d_lane]; float dq_o_1 = dq2_1 + dqf_1; - float dk_o_1 = dk2b_1 + dkf_1 + dkt_0[6]; - float dg_o_1 = __fmaf_rn(q_t_1_1, dq2_1, (dk2b_1 - dkt_0[6]) * k_t_1) + dgf_1; - s_run[0] = s_run[0] + dg_o_1; + float dk_o_1 = dk2b_1 + dkf_1 + dkt_1[6]; + float dg_o_1 = __fmaf_rn(q_t_1_1_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_1[6], k_t_1, cr_0)) + dgf_1; + s_run_0[0] = s_run_0[0] + dg_o_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 6)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_1 = t_1 * qrs; if (t_1 < clen) { { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_261 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_262 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dk_o_1; } } - int t_2 = 32 + 8 * th + 5; - float r_254[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_2 = r_254[0]; - unsigned int r_255[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((kb8 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_2 = __uint_as_float(r_255[0] << 16); - unsigned int r_256[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_2 = __uint_as_float(r_256[0] << 16); + int t_2 = 48 + 8 * th + 5; + float r_258[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_258[0]; + unsigned int r_259[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_259[0] << 16); + unsigned int r_260[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_260[0] << 16); float q_t_1_2 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew + 640 + d_lane]; - float dkf_2 = pool_f32[ew + 1664 + d_lane]; - float dgf_2 = pool_f32[ew + 2688 + d_lane]; - float bt_t_2 = btv[5]; - float e_off_2 = approx_exp2(g_t_2 - gn); - float e_dg_2 = approx_exp2(g_t_2 - gc_0); - float dq2_2 = __fmaf_rn(qo[5], e_off_2, qdg[5] * e_dg_2); - float dk2_2 = __fmaf_rn(ko[5], e_off_2, kdg[5] * e_dg_2); - dbp[5] = dk2_2 * k_t_2; + float bt_t_2 = btv_1[5]; + float e_off_2 = approx_exp2(g_t_2 - gn_0); + float e_dg_2 = approx_exp2(g_t_2 - gc_1); + float dq2_2 = __fmaf_rn(qo_1[5], e_off_2, qdg_1[5] * e_dg_2); + float dk2_2 = __fmaf_rn(ko_1[5], e_off_2, kdg_1[5] * e_dg_2); + dbp_0[5] = dk2_2 * k_t_2; float dk2b_2 = dk2_2 * bt_t_2; + float qe_1_1 = q_t_1_2 * e_dg_2; + float kp_13 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1_1 = (__nv_bfloat16)kp_13; + float kbe_1_1 = (float)kb_1_1 * e_dg_2; + float cr_1_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1_1) & 4294959104u) - qe_1_1, qdg_1[5], (__uint_as_float(__as_u32(kbe_1_1) & 4294959104u) - kp_13 * e_dg_2) * kdg_1[5]); + float dqf_2 = pool_f32[ew_0 + 640 + d_lane]; + float dkf_2 = pool_f32[ew_0 + 1664 + d_lane]; + float dgf_2 = pool_f32[ew_0 + 2688 + d_lane]; float dq_o_2 = dq2_2 + dqf_2; - float dk_o_2 = dk2b_2 + dkf_2 + dkt_0[5]; - float dg_o_2 = __fmaf_rn(q_t_1_2, dq2_2, (dk2b_2 - dkt_0[5]) * k_t_2) + dgf_2; - s_run[0] = s_run[0] + dg_o_2; + float dk_o_2 = dk2b_2 + dkf_2 + dkt_1[5]; + float dg_o_2 = __fmaf_rn(q_t_1_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_1[5], k_t_2, cr_1_1)) + dgf_2; + s_run_0[0] = s_run_0[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 5)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_263 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_264 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_264)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_264) + (0)) = (__nv_bfloat16)dk_o_2; } } - int t_3 = 32 + 8 * th + 4; - float r_257[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_3 = r_257[0]; - unsigned int r_258[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((kb8 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_3 = __uint_as_float(r_258[0] << 16); - unsigned int r_259[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_3 = __uint_as_float(r_259[0] << 16); + int t_3 = 48 + 8 * th + 4; + float r_261[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_3 = r_261[0]; + unsigned int r_262[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_3 = __uint_as_float(r_262[0] << 16); + unsigned int r_263[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_3 = __uint_as_float(r_263[0] << 16); float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew + 512 + d_lane]; - float dkf_3 = pool_f32[ew + 1536 + d_lane]; - float dgf_3 = pool_f32[ew + 2560 + d_lane]; - float bt_t_3 = btv[4]; - float e_off_3 = approx_exp2(g_t_3 - gn); - float e_dg_3 = approx_exp2(g_t_3 - gc_0); - float dq2_3 = __fmaf_rn(qo[4], e_off_3, qdg[4] * e_dg_3); - float dk2_3 = __fmaf_rn(ko[4], e_off_3, kdg[4] * e_dg_3); - dbp[4] = dk2_3 * k_t_3; + float bt_t_3 = btv_1[4]; + float e_off_3 = approx_exp2(g_t_3 - gn_0); + float e_dg_3 = approx_exp2(g_t_3 - gc_1); + float dq2_3 = __fmaf_rn(qo_1[4], e_off_3, qdg_1[4] * e_dg_3); + float dk2_3 = __fmaf_rn(ko_1[4], e_off_3, kdg_1[4] * e_dg_3); + dbp_0[4] = dk2_3 * k_t_3; float dk2b_3 = dk2_3 * bt_t_3; + float qe_2 = q_t_1_3 * e_dg_3; + float kp_14 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_14; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg_1[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_14 * e_dg_3) * kdg_1[4]); + float dqf_3 = pool_f32[ew_0 + 512 + d_lane]; + float dkf_3 = pool_f32[ew_0 + 1536 + d_lane]; + float dgf_3 = pool_f32[ew_0 + 2560 + d_lane]; float dq_o_3 = dq2_3 + dqf_3; - float dk_o_3 = dk2b_3 + dkf_3 + dkt_0[4]; - float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, (dk2b_3 - dkt_0[4]) * k_t_3) + dgf_3; - s_run[0] = s_run[0] + dg_o_3; + float dk_o_3 = dk2b_3 + dkf_3 + dkt_1[4]; + float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_1[4], k_t_3, cr_2)) + dgf_3; + s_run_0[0] = s_run_0[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 4)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_265 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_265)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_266 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_266)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_266) + (0)) = (__nv_bfloat16)dk_o_3; } } - int t_4 = 32 + 8 * th + 3; - float r_260[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_4 = r_260[0]; - unsigned int r_261[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((kb8 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_4 = __uint_as_float(r_261[0] << 16); - unsigned int r_262[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_4 = __uint_as_float(r_262[0] << 16); + int t_4 = 48 + 8 * th + 3; + float r_264[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_4 = r_264[0]; + unsigned int r_265[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_4 = __uint_as_float(r_265[0] << 16); + unsigned int r_266[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_4 = __uint_as_float(r_266[0] << 16); float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew + 384 + d_lane]; - float dkf_4 = pool_f32[ew + 1408 + d_lane]; - float dgf_4 = pool_f32[ew + 2432 + d_lane]; - float bt_t_4 = btv[3]; - float e_off_4 = approx_exp2(g_t_4 - gn); - float e_dg_4 = approx_exp2(g_t_4 - gc_0); - float dq2_4 = __fmaf_rn(qo[3], e_off_4, qdg[3] * e_dg_4); - float dk2_4 = __fmaf_rn(ko[3], e_off_4, kdg[3] * e_dg_4); - dbp[3] = dk2_4 * k_t_4; + float bt_t_4 = btv_1[3]; + float e_off_4 = approx_exp2(g_t_4 - gn_0); + float e_dg_4 = approx_exp2(g_t_4 - gc_1); + float dq2_4 = __fmaf_rn(qo_1[3], e_off_4, qdg_1[3] * e_dg_4); + float dk2_4 = __fmaf_rn(ko_1[3], e_off_4, kdg_1[3] * e_dg_4); + dbp_0[3] = dk2_4 * k_t_4; float dk2b_4 = dk2_4 * bt_t_4; + float qe_3 = q_t_1_4 * e_dg_4; + float kp_15 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg_1[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_4) * kdg_1[3]); + float dqf_4 = pool_f32[ew_0 + 384 + d_lane]; + float dkf_4 = pool_f32[ew_0 + 1408 + d_lane]; + float dgf_4 = pool_f32[ew_0 + 2432 + d_lane]; float dq_o_4 = dq2_4 + dqf_4; - float dk_o_4 = dk2b_4 + dkf_4 + dkt_0[3]; - float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, (dk2b_4 - dkt_0[3]) * k_t_4) + dgf_4; - s_run[0] = s_run[0] + dg_o_4; + float dk_o_4 = dk2b_4 + dkf_4 + dkt_1[3]; + float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_1[3], k_t_4, cr_3)) + dgf_4; + s_run_0[0] = s_run_0[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 3)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_267 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_267)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_267) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_268 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_268)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dk_o_4; } } - int t_5 = 32 + 8 * th + 2; - float r_263[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_5 = r_263[0]; - unsigned int r_264[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((kb8 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_5 = __uint_as_float(r_264[0] << 16); - unsigned int r_265[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_5 = __uint_as_float(r_265[0] << 16); + int t_5 = 48 + 8 * th + 2; + float r_267[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_5 = r_267[0]; + unsigned int r_268[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_5 = __uint_as_float(r_268[0] << 16); + unsigned int r_269[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_5 = __uint_as_float(r_269[0] << 16); float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew + 256 + d_lane]; - float dkf_5 = pool_f32[ew + 1280 + d_lane]; - float dgf_5 = pool_f32[ew + 2304 + d_lane]; - float bt_t_5 = btv[2]; - float e_off_5 = approx_exp2(g_t_5 - gn); - float e_dg_5 = approx_exp2(g_t_5 - gc_0); - float dq2_5 = __fmaf_rn(qo[2], e_off_5, qdg[2] * e_dg_5); - float dk2_5 = __fmaf_rn(ko[2], e_off_5, kdg[2] * e_dg_5); - dbp[2] = dk2_5 * k_t_5; + float bt_t_5 = btv_1[2]; + float e_off_5 = approx_exp2(g_t_5 - gn_0); + float e_dg_5 = approx_exp2(g_t_5 - gc_1); + float dq2_5 = __fmaf_rn(qo_1[2], e_off_5, qdg_1[2] * e_dg_5); + float dk2_5 = __fmaf_rn(ko_1[2], e_off_5, kdg_1[2] * e_dg_5); + dbp_0[2] = dk2_5 * k_t_5; float dk2b_5 = dk2_5 * bt_t_5; + float qe_4 = q_t_1_5 * e_dg_5; + float kp_16 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg_1[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_5) * kdg_1[2]); + float dqf_5 = pool_f32[ew_0 + 256 + d_lane]; + float dkf_5 = pool_f32[ew_0 + 1280 + d_lane]; + float dgf_5 = pool_f32[ew_0 + 2304 + d_lane]; float dq_o_5 = dq2_5 + dqf_5; - float dk_o_5 = dk2b_5 + dkf_5 + dkt_0[2]; - float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, (dk2b_5 - dkt_0[2]) * k_t_5) + dgf_5; - s_run[0] = s_run[0] + dg_o_5; + float dk_o_5 = dk2b_5 + dkf_5 + dkt_1[2]; + float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_1[2], k_t_5, cr_4)) + dgf_5; + s_run_0[0] = s_run_0[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 2)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_269 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_269)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_269) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_270 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_270)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_270) + (0)) = (__nv_bfloat16)dk_o_5; } } - int t_6 = 32 + 8 * th + 1; - float r_266[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_6 = r_266[0]; - unsigned int r_267[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((kb8 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_6 = __uint_as_float(r_267[0] << 16); - unsigned int r_268[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_6 = __uint_as_float(r_268[0] << 16); + int t_6 = 48 + 8 * th + 1; + float r_270[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_6 = r_270[0]; + unsigned int r_271[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_6 = __uint_as_float(r_271[0] << 16); + unsigned int r_272[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_6 = __uint_as_float(r_272[0] << 16); float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew + 128 + d_lane]; - float dkf_6 = pool_f32[ew + 1152 + d_lane]; - float dgf_6 = pool_f32[ew + 2176 + d_lane]; - float bt_t_6 = btv[1]; - float e_off_6 = approx_exp2(g_t_6 - gn); - float e_dg_6 = approx_exp2(g_t_6 - gc_0); - float dq2_6 = __fmaf_rn(qo[1], e_off_6, qdg[1] * e_dg_6); - float dk2_6 = __fmaf_rn(ko[1], e_off_6, kdg[1] * e_dg_6); - dbp[1] = dk2_6 * k_t_6; + float bt_t_6 = btv_1[1]; + float e_off_6 = approx_exp2(g_t_6 - gn_0); + float e_dg_6 = approx_exp2(g_t_6 - gc_1); + float dq2_6 = __fmaf_rn(qo_1[1], e_off_6, qdg_1[1] * e_dg_6); + float dk2_6 = __fmaf_rn(ko_1[1], e_off_6, kdg_1[1] * e_dg_6); + dbp_0[1] = dk2_6 * k_t_6; float dk2b_6 = dk2_6 * bt_t_6; + float qe_5 = q_t_1_6 * e_dg_6; + float kp_17 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg_1[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_6) * kdg_1[1]); + float dqf_6 = pool_f32[ew_0 + 128 + d_lane]; + float dkf_6 = pool_f32[ew_0 + 1152 + d_lane]; + float dgf_6 = pool_f32[ew_0 + 2176 + d_lane]; float dq_o_6 = dq2_6 + dqf_6; - float dk_o_6 = dk2b_6 + dkf_6 + dkt_0[1]; - float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, (dk2b_6 - dkt_0[1]) * k_t_6) + dgf_6; - s_run[0] = s_run[0] + dg_o_6; + float dk_o_6 = dk2b_6 + dkf_6 + dkt_1[1]; + float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_1[1], k_t_6, cr_5)) + dgf_6; + s_run_0[0] = s_run_0[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th + 1)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_271 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_271)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_272 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_272)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_272) + (0)) = (__nv_bfloat16)dk_o_6; } } - int t_7 = 32 + 8 * th; - float r_269[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((gb8 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_7 = r_269[0]; - unsigned int r_270[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((kb8 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_7 = __uint_as_float(r_270[0] << 16); - unsigned int r_271[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((qbase_0 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_7 = __uint_as_float(r_271[0] << 16); + int t_7 = 48 + 8 * th; + float r_273[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_7 = r_273[0]; + unsigned int r_274[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_7 = __uint_as_float(r_274[0] << 16); + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_7 = __uint_as_float(r_275[0] << 16); float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew + d_lane]; - float dkf_7 = pool_f32[ew + 1024 + d_lane]; - float dgf_7 = pool_f32[ew + 2048 + d_lane]; - float bt_t_7 = btv[0]; - float e_off_7 = approx_exp2(g_t_7 - gn); - float e_dg_7 = approx_exp2(g_t_7 - gc_0); - float dq2_7 = __fmaf_rn(qo[0], e_off_7, qdg[0] * e_dg_7); - float dk2_7 = __fmaf_rn(ko[0], e_off_7, kdg[0] * e_dg_7); - dbp[0] = dk2_7 * k_t_7; + float bt_t_7 = btv_1[0]; + float e_off_7 = approx_exp2(g_t_7 - gn_0); + float e_dg_7 = approx_exp2(g_t_7 - gc_1); + float dq2_7 = __fmaf_rn(qo_1[0], e_off_7, qdg_1[0] * e_dg_7); + float dk2_7 = __fmaf_rn(ko_1[0], e_off_7, kdg_1[0] * e_dg_7); + dbp_0[0] = dk2_7 * k_t_7; float dk2b_7 = dk2_7 * bt_t_7; + float qe_6 = q_t_1_7 * e_dg_7; + float kp_18 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg_1[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_7) * kdg_1[0]); + float dqf_7 = pool_f32[ew_0 + d_lane]; + float dkf_7 = pool_f32[ew_0 + 1024 + d_lane]; + float dgf_7 = pool_f32[ew_0 + 2048 + d_lane]; float dq_o_7 = dq2_7 + dqf_7; - float dk_o_7 = dk2b_7 + dkf_7 + dkt_0[0]; - float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, (dk2b_7 - dkt_0[0]) * k_t_7) + dgf_7; - s_run[0] = s_run[0] + dg_o_7; + float dk_o_7 = dk2b_7 + dkf_7 + dkt_1[0]; + float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_1[0], k_t_7, cr_6)) + dgf_7; + s_run_0[0] = s_run_0[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(192 + 8 * th)), "f"(s_run[0])); + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_0[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_273 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_274 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_7; +#else + #pragma unroll 1 + for (int k_1 = 0; k_1 < 8; k_1++) { + int j_1 = 7 - k_1; + float qo_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qo_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th + j_1))); + float ko_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(ko_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th + j_1))); + float qdg_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(qdg_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + j_1))); + float kdg_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(kdg_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th + j_1))); + float dkt_1[1]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x1.b32" + " {%0}, [%1];" + : "=f"(dkt_1[0]) + : "r"(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th + j_1))); + float btv_1[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&btv_1[0])) : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th + j_1)))); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); + int t_1 = sb8_0 + j_1; + float r_252[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_252[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 * j_1 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1 = r_252[0]; + unsigned int r_253[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1 = __uint_as_float(r_253[0] << 16); + unsigned int r_254[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_254[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 * j_1 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2 = __uint_as_float(r_254[0] << 16); + float q_t_1_1 = ((t_1 >= clen) ? 0.0f : q_t_2); + float bt_t_1 = btv_1[0]; + float e_off_1 = approx_exp2(g_t_1 - gn_0); + float e_dg_1 = approx_exp2(g_t_1 - gc_1); + float dq2_1 = __fmaf_rn(qo_1[0], e_off_1, qdg_1[0] * e_dg_1); + float dk2_1 = __fmaf_rn(ko_1[0], e_off_1, kdg_1[0] * e_dg_1); + float dbv_1[1]; + dbv_1[0] = dk2_1 * k_t_1; + float dk2b_1 = dk2_1 * bt_t_1; + float qe_1 = q_t_1_1 * e_dg_1; + float kp_12 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_12; + float kbe_1 = (float)kb_1 * e_dg_1; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg_1[0], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_12 * e_dg_1) * kdg_1[0]); + float dqf_1 = pool_f32[ew_0 + 128 * j_1 + d_lane]; + float dkf_1 = pool_f32[ew_0 + (1024 + 128 * j_1) + d_lane]; + float dgf_1 = pool_f32[ew_0 + (2048 + 128 * j_1) + d_lane]; + float dq_o_1 = dq2_1 + dqf_1; + float dk_o_1 = dk2b_1 + dkf_1 + dkt_1[0]; + float dg_o_1 = __fmaf_rn(q_t_1_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_1[0], k_t_1, cr_1)) + dgf_1; + s_run_0[0] = s_run_0[0] + dg_o_1; + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + j_1)), "f"(s_run_0[0])); + asm volatile( + "tcgen05.st.sync.aligned.32x32b.x1.b32" + " [%0], {%1};" + :: "r"(taddr + (unsigned int)row_base + (unsigned int)(c_bp_0 + j_1)), "f"(dbv_1[0])); + asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); + int qoff_1 = t_1 * qrs; + if (t_1 < clen) { + { + auto _gst_259 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_1; + } + { + auto _gst_260 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_1; + } +#endif } } - float tot = s_run[0]; - int b4 = lane_1 >> 4 & 1; - int b3 = lane_1 >> 3 & 1; - int b2 = lane_1 >> 2 & 1; - float v4[4]; - float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; - float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; - float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; - float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; - float v2[2]; - float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; - float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; - float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float tot_0 = s_run_0[0]; +#if __CUDA_ARCH__ == 1030 +#else + float dbp_0[8]; + tmem_ld_x8(&dbp_0[0], taddr + (unsigned int)row_base + (unsigned int)c_bp_0); + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); +#endif + int b4_0 = lane_1 >> 4 & 1; + int b3_0 = lane_1 >> 3 & 1; + int b2_0 = lane_1 >> 2 & 1; + float v4_0[4]; + float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_275; +#else + float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_261; +#endif + float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_276; +#else + float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_262; +#endif + float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_277; +#else + float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_263; +#endif + float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_278; +#else + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_264; +#endif + float v2_16[2]; + float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_279; +#else + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_265; +#endif + float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_280; +#else + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_266; +#endif + float send_12 = ((b2_0 == 0) ? v2_16[1] : v2_16[0]); +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_281; + float _shfl_xor_sync_282 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_282; + float _shfl_xor_sync_283 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_283; +#else + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_267; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_268; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_269; +#endif asm volatile("tcgen05.fence::before_thread_sync;"); - mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); - int pf_row = 32 + 8 * th + (lane_1 & 7) + 16 * (lane_1 >> 3); - long long pf_off = (ext0 + (long long)pf_row) * g_raw_row_stride + (long long)hv * g_raw_head_stride; - if (lane_1 < 16 && pf_row < clen) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(g_raw + (pf_off + (long long)(32 * qd))))); } - if (lane_1 == 16) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(A_log + hv))); } - if (lane_1 == 17) { asm volatile("prefetch.global.L2 [%0];" :: "l"((uint64_t)(dt_bias + (hv * 128 + 32 * qd)))); } - mbarrier_wait(epi_full_addr + (th) * 8, 1); - asm volatile("tcgen05.fence::after_thread_sync;"); - float r_272[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 6144 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_0 = r_272[0]; - float gc_1 = gn_0 - 21.640425f; - int eb_0 = 98304 + th * -32768; - int ew_0 = eb_0 >> 2; - int qbase_1 = pool_f32_addr + (unsigned int)(eb_0 + 12288 + dh_lane * 1024); - int sb8_0 = 48 + 8 * th; - int gb8_0 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_0 * 128); - int kb8_0 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_0 * 128); - float dbp_0[8]; - float qo_0[8]; - tmem_ld_x8(&qo_0[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); - float ko_0[8]; - tmem_ld_x8(&ko_0[0], taddr + (unsigned int)row_base + (unsigned int)(176 + 8 * th)); - float qdg_0[8]; - tmem_ld_x8(&qdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)); - float kdg_0[8]; - tmem_ld_x8(&kdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(240 + 8 * th)); - float dkt_1[8]; - tmem_ld_x8(&dkt_1[0], taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th)); - float btv_0[8]; - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_0[0])), "=r"(*reinterpret_cast(&btv_0[(0) + 1])), "=r"(*reinterpret_cast(&btv_0[(0) + 2])), "=r"(*reinterpret_cast(&btv_0[(0) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th)))); - asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_0[4])), "=r"(*reinterpret_cast(&btv_0[(4) + 1])), "=r"(*reinterpret_cast(&btv_0[(4) + 2])), "=r"(*reinterpret_cast(&btv_0[(4) + 3])) - : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (48 + 8 * th) + 16))); - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float s_run_0[1]; - s_run_0[0] = 0.0f; - int t_8 = 48 + 8 * th + 7; - float r_273[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_8 = r_273[0]; - unsigned int r_274[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_8 = __uint_as_float(r_274[0] << 16); - unsigned int r_275[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_8 = __uint_as_float(r_275[0] << 16); - float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_0 + 896 + d_lane]; - float dkf_8 = pool_f32[ew_0 + 1920 + d_lane]; - float dgf_8 = pool_f32[ew_0 + 2944 + d_lane]; - float bt_t_8 = btv_0[7]; - float e_off_8 = approx_exp2(g_t_8 - gn_0); - float e_dg_8 = approx_exp2(g_t_8 - gc_1); - float dq2_8 = __fmaf_rn(qo_0[7], e_off_8, qdg_0[7] * e_dg_8); - float dk2_8 = __fmaf_rn(ko_0[7], e_off_8, kdg_0[7] * e_dg_8); - dbp_0[7] = dk2_8 * k_t_8; - float dk2b_8 = dk2_8 * bt_t_8; - float dq_o_8 = dq2_8 + dqf_8; - float dk_o_8 = dk2b_8 + dkf_8 + dkt_1[7]; - float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, (dk2b_8 - dkt_1[7]) * k_t_8) + dgf_8; - s_run_0[0] = s_run_0[0] + dg_o_8; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 7)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_8 = t_8 * qrs; - if (t_8 < clen) { - { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; - } - { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; - } - } - int t_9 = 48 + 8 * th + 6; - float r_276[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_9 = r_276[0]; - unsigned int r_277[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_9 = __uint_as_float(r_277[0] << 16); - unsigned int r_278[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_9 = __uint_as_float(r_278[0] << 16); - float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_0 + 768 + d_lane]; - float dkf_9 = pool_f32[ew_0 + 1792 + d_lane]; - float dgf_9 = pool_f32[ew_0 + 2816 + d_lane]; - float bt_t_9 = btv_0[6]; - float e_off_9 = approx_exp2(g_t_9 - gn_0); - float e_dg_9 = approx_exp2(g_t_9 - gc_1); - float dq2_9 = __fmaf_rn(qo_0[6], e_off_9, qdg_0[6] * e_dg_9); - float dk2_9 = __fmaf_rn(ko_0[6], e_off_9, kdg_0[6] * e_dg_9); - dbp_0[6] = dk2_9 * k_t_9; - float dk2b_9 = dk2_9 * bt_t_9; - float dq_o_9 = dq2_9 + dqf_9; - float dk_o_9 = dk2b_9 + dkf_9 + dkt_1[6]; - float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, (dk2b_9 - dkt_1[6]) * k_t_9) + dgf_9; - s_run_0[0] = s_run_0[0] + dg_o_9; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 6)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_9 = t_9 * qrs; - if (t_9 < clen) { - { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; - } - { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; - } - } - int t_10 = 48 + 8 * th + 5; - float r_279[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_10 = r_279[0]; - unsigned int r_280[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_10 = __uint_as_float(r_280[0] << 16); - unsigned int r_281[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_10 = __uint_as_float(r_281[0] << 16); - float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_0 + 640 + d_lane]; - float dkf_10 = pool_f32[ew_0 + 1664 + d_lane]; - float dgf_10 = pool_f32[ew_0 + 2688 + d_lane]; - float bt_t_10 = btv_0[5]; - float e_off_10 = approx_exp2(g_t_10 - gn_0); - float e_dg_10 = approx_exp2(g_t_10 - gc_1); - float dq2_10 = __fmaf_rn(qo_0[5], e_off_10, qdg_0[5] * e_dg_10); - float dk2_10 = __fmaf_rn(ko_0[5], e_off_10, kdg_0[5] * e_dg_10); - dbp_0[5] = dk2_10 * k_t_10; - float dk2b_10 = dk2_10 * bt_t_10; - float dq_o_10 = dq2_10 + dqf_10; - float dk_o_10 = dk2b_10 + dkf_10 + dkt_1[5]; - float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, (dk2b_10 - dkt_1[5]) * k_t_10) + dgf_10; - s_run_0[0] = s_run_0[0] + dg_o_10; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 5)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_10 = t_10 * qrs; - if (t_10 < clen) { - { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; - } - { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; - } - } - int t_11 = 48 + 8 * th + 4; - float r_282[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_11 = r_282[0]; - unsigned int r_283[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_11 = __uint_as_float(r_283[0] << 16); - unsigned int r_284[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_11 = __uint_as_float(r_284[0] << 16); - float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_0 + 512 + d_lane]; - float dkf_11 = pool_f32[ew_0 + 1536 + d_lane]; - float dgf_11 = pool_f32[ew_0 + 2560 + d_lane]; - float bt_t_11 = btv_0[4]; - float e_off_11 = approx_exp2(g_t_11 - gn_0); - float e_dg_11 = approx_exp2(g_t_11 - gc_1); - float dq2_11 = __fmaf_rn(qo_0[4], e_off_11, qdg_0[4] * e_dg_11); - float dk2_11 = __fmaf_rn(ko_0[4], e_off_11, kdg_0[4] * e_dg_11); - dbp_0[4] = dk2_11 * k_t_11; - float dk2b_11 = dk2_11 * bt_t_11; - float dq_o_11 = dq2_11 + dqf_11; - float dk_o_11 = dk2b_11 + dkf_11 + dkt_1[4]; - float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, (dk2b_11 - dkt_1[4]) * k_t_11) + dgf_11; - s_run_0[0] = s_run_0[0] + dg_o_11; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 4)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_11 = t_11 * qrs; - if (t_11 < clen) { - { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; - } - { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; - } - } - int t_12 = 48 + 8 * th + 3; - float r_285[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_12 = r_285[0]; - unsigned int r_286[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_12 = __uint_as_float(r_286[0] << 16); - unsigned int r_287[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_12 = __uint_as_float(r_287[0] << 16); - float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_0 + 384 + d_lane]; - float dkf_12 = pool_f32[ew_0 + 1408 + d_lane]; - float dgf_12 = pool_f32[ew_0 + 2432 + d_lane]; - float bt_t_12 = btv_0[3]; - float e_off_12 = approx_exp2(g_t_12 - gn_0); - float e_dg_12 = approx_exp2(g_t_12 - gc_1); - float dq2_12 = __fmaf_rn(qo_0[3], e_off_12, qdg_0[3] * e_dg_12); - float dk2_12 = __fmaf_rn(ko_0[3], e_off_12, kdg_0[3] * e_dg_12); - dbp_0[3] = dk2_12 * k_t_12; - float dk2b_12 = dk2_12 * bt_t_12; - float dq_o_12 = dq2_12 + dqf_12; - float dk_o_12 = dk2b_12 + dkf_12 + dkt_1[3]; - float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, (dk2b_12 - dkt_1[3]) * k_t_12) + dgf_12; - s_run_0[0] = s_run_0[0] + dg_o_12; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 3)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_12 = t_12 * qrs; - if (t_12 < clen) { - { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; - } - { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; - } - } - int t_13 = 48 + 8 * th + 2; - float r_288[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_13 = r_288[0]; - unsigned int r_289[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_13 = __uint_as_float(r_289[0] << 16); - unsigned int r_290[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_13 = __uint_as_float(r_290[0] << 16); - float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_0 + 256 + d_lane]; - float dkf_13 = pool_f32[ew_0 + 1280 + d_lane]; - float dgf_13 = pool_f32[ew_0 + 2304 + d_lane]; - float bt_t_13 = btv_0[2]; - float e_off_13 = approx_exp2(g_t_13 - gn_0); - float e_dg_13 = approx_exp2(g_t_13 - gc_1); - float dq2_13 = __fmaf_rn(qo_0[2], e_off_13, qdg_0[2] * e_dg_13); - float dk2_13 = __fmaf_rn(ko_0[2], e_off_13, kdg_0[2] * e_dg_13); - dbp_0[2] = dk2_13 * k_t_13; - float dk2b_13 = dk2_13 * bt_t_13; - float dq_o_13 = dq2_13 + dqf_13; - float dk_o_13 = dk2b_13 + dkf_13 + dkt_1[2]; - float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, (dk2b_13 - dkt_1[2]) * k_t_13) + dgf_13; - s_run_0[0] = s_run_0[0] + dg_o_13; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 2)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_13 = t_13 * qrs; - if (t_13 < clen) { - { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; - } - { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; - } - } - int t_14 = 48 + 8 * th + 1; - float r_291[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_14 = r_291[0]; - unsigned int r_292[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_14 = __uint_as_float(r_292[0] << 16); - unsigned int r_293[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_14 = __uint_as_float(r_293[0] << 16); - float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_0 + 128 + d_lane]; - float dkf_14 = pool_f32[ew_0 + 1152 + d_lane]; - float dgf_14 = pool_f32[ew_0 + 2176 + d_lane]; - float bt_t_14 = btv_0[1]; - float e_off_14 = approx_exp2(g_t_14 - gn_0); - float e_dg_14 = approx_exp2(g_t_14 - gc_1); - float dq2_14 = __fmaf_rn(qo_0[1], e_off_14, qdg_0[1] * e_dg_14); - float dk2_14 = __fmaf_rn(ko_0[1], e_off_14, kdg_0[1] * e_dg_14); - dbp_0[1] = dk2_14 * k_t_14; - float dk2b_14 = dk2_14 * bt_t_14; - float dq_o_14 = dq2_14 + dqf_14; - float dk_o_14 = dk2b_14 + dkf_14 + dkt_1[1]; - float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, (dk2b_14 - dkt_1[1]) * k_t_14) + dgf_14; - s_run_0[0] = s_run_0[0] + dg_o_14; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th + 1)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_14 = t_14 * qrs; - if (t_14 < clen) { - { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; - } - { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; - } - } - int t_15 = 48 + 8 * th; - float r_294[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((gb8_0 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_15 = r_294[0]; - unsigned int r_295[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((kb8_0 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_15 = __uint_as_float(r_295[0] << 16); - unsigned int r_296[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_1 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_15 = __uint_as_float(r_296[0] << 16); - float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_0 + d_lane]; - float dkf_15 = pool_f32[ew_0 + 1024 + d_lane]; - float dgf_15 = pool_f32[ew_0 + 2048 + d_lane]; - float bt_t_15 = btv_0[0]; - float e_off_15 = approx_exp2(g_t_15 - gn_0); - float e_dg_15 = approx_exp2(g_t_15 - gc_1); - float dq2_15 = __fmaf_rn(qo_0[0], e_off_15, qdg_0[0] * e_dg_15); - float dk2_15 = __fmaf_rn(ko_0[0], e_off_15, kdg_0[0] * e_dg_15); - dbp_0[0] = dk2_15 * k_t_15; - float dk2b_15 = dk2_15 * bt_t_15; - float dq_o_15 = dq2_15 + dqf_15; - float dk_o_15 = dk2b_15 + dkf_15 + dkt_1[0]; - float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, (dk2b_15 - dkt_1[0]) * k_t_15) + dgf_15; - s_run_0[0] = s_run_0[0] + dg_o_15; - asm volatile( - "tcgen05.st.sync.aligned.32x32b.x1.b32" - " [%0], {%1};" - :: "r"(taddr + (unsigned int)row_base + (unsigned int)(224 + 8 * th)), "f"(s_run_0[0])); - asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_15 = t_15 * qrs; - if (t_15 < clen) { - { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; - } - { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; - } - } - float tot_0 = s_run_0[0]; - int b4_0 = lane_1 >> 4 & 1; - int b3_0 = lane_1 >> 3 & 1; - int b2_0 = lane_1 >> 2 & 1; - float v4_0[4]; - float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; - float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; - float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; - float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; - float v2_16[2]; - float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_16[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; - float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_16[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; - float send_12 = ((b2_0 == 0) ? v2_16[1] : v2_16[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_16[0] : v2_16[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; - asm volatile("tcgen05.fence::before_thread_sync;"); - mbarrier_arrive(epi_empty_addr + (th) * 8); - float gacc[8]; - float rg_rate = __nv_expf(A_log[hv]); - float rg_clr; - asm("mul.rn.ftz.f32 %0, %1, %2;" : "=f"(rg_clr) : "f"(lower_bound), "f"(rg_rate)); - gacc[0] = 0.0f; - gacc[1] = 0.0f; - gacc[2] = 0.0f; - gacc[3] = 0.0f; - gacc[4] = rg_rate; - gacc[5] = rg_clr; - gacc[6] = dt_bias[hv * 128 + d_lane]; - gacc[7] = 0.0f; - float raw_t3[8]; - long long rb = ext0 * g_raw_row_stride + (long long)hv * g_raw_head_stride + (long long)d_lane; - int t_c_0 = ((48 + 8 * th < clen) ? 48 + 8 * th : clen - 1); - raw_t3[0] = (float)g_raw[rb + (long long)t_c_0 * g_raw_row_stride]; - int t_c_1 = ((48 + 8 * th + 1 < clen) ? 48 + 8 * th + 1 : clen - 1); - raw_t3[1] = (float)g_raw[rb + (long long)t_c_1 * g_raw_row_stride]; - int t_c_2 = ((48 + 8 * th + 2 < clen) ? 48 + 8 * th + 2 : clen - 1); - raw_t3[2] = (float)g_raw[rb + (long long)t_c_2 * g_raw_row_stride]; - int t_c_3 = ((48 + 8 * th + 3 < clen) ? 48 + 8 * th + 3 : clen - 1); - raw_t3[3] = (float)g_raw[rb + (long long)t_c_3 * g_raw_row_stride]; - int t_c_4 = ((48 + 8 * th + 4 < clen) ? 48 + 8 * th + 4 : clen - 1); - raw_t3[4] = (float)g_raw[rb + (long long)t_c_4 * g_raw_row_stride]; - int t_c_5 = ((48 + 8 * th + 5 < clen) ? 48 + 8 * th + 5 : clen - 1); - raw_t3[5] = (float)g_raw[rb + (long long)t_c_5 * g_raw_row_stride]; - int t_c_6 = ((48 + 8 * th + 6 < clen) ? 48 + 8 * th + 6 : clen - 1); - raw_t3[6] = (float)g_raw[rb + (long long)t_c_6 * g_raw_row_stride]; - int t_c_7 = ((48 + 8 * th + 7 < clen) ? 48 + 8 * th + 7 : clen - 1); - raw_t3[7] = (float)g_raw[rb + (long long)t_c_7 * g_raw_row_stride]; - float raw_t2[8]; - long long rb_0 = ext0 * g_raw_row_stride + (long long)hv * g_raw_head_stride + (long long)d_lane; - int t_c_8 = ((32 + 8 * th < clen) ? 32 + 8 * th : clen - 1); - raw_t2[0] = (float)g_raw[rb_0 + (long long)t_c_8 * g_raw_row_stride]; - int t_c_9 = ((32 + 8 * th + 1 < clen) ? 32 + 8 * th + 1 : clen - 1); - raw_t2[1] = (float)g_raw[rb_0 + (long long)t_c_9 * g_raw_row_stride]; - int t_c_10 = ((32 + 8 * th + 2 < clen) ? 32 + 8 * th + 2 : clen - 1); - raw_t2[2] = (float)g_raw[rb_0 + (long long)t_c_10 * g_raw_row_stride]; - int t_c_11 = ((32 + 8 * th + 3 < clen) ? 32 + 8 * th + 3 : clen - 1); - raw_t2[3] = (float)g_raw[rb_0 + (long long)t_c_11 * g_raw_row_stride]; - int t_c_12 = ((32 + 8 * th + 4 < clen) ? 32 + 8 * th + 4 : clen - 1); - raw_t2[4] = (float)g_raw[rb_0 + (long long)t_c_12 * g_raw_row_stride]; - int t_c_13 = ((32 + 8 * th + 5 < clen) ? 32 + 8 * th + 5 : clen - 1); - raw_t2[5] = (float)g_raw[rb_0 + (long long)t_c_13 * g_raw_row_stride]; - int t_c_14 = ((32 + 8 * th + 6 < clen) ? 32 + 8 * th + 6 : clen - 1); - raw_t2[6] = (float)g_raw[rb_0 + (long long)t_c_14 * g_raw_row_stride]; - int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); - raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; - { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); - } - asm volatile("tcgen05.fence::before_thread_sync;"); - if (qd == 0) { - asm volatile("barrier.sync 2, 64;" ::: "memory"); - } - if (qd == 1) { - asm volatile("barrier.sync 3, 64;" ::: "memory"); - } - if (qd == 2) { - asm volatile("barrier.sync 4, 64;" ::: "memory"); - } - if (qd == 3) { - asm volatile("barrier.sync 5, 64;" ::: "memory"); - } + float gacc[8]; + float rg_rate = __nv_expf(A_log[hv]); + float rg_clr; + asm("mul.rn.ftz.f32 %0, %1, %2;" : "=f"(rg_clr) : "f"(lower_bound), "f"(rg_rate)); + gacc[0] = 0.0f; + gacc[1] = 0.0f; + gacc[2] = 0.0f; + gacc[3] = 0.0f; + gacc[4] = rg_rate; + gacc[5] = rg_clr; + gacc[6] = dt_bias[hv * 128 + d_lane]; + gacc[7] = 0.0f; + float raw_t3[8]; + long long rb = ext0 * g_raw_row_stride + (long long)hv * g_raw_head_stride + (long long)d_lane; + int t_c_0 = ((48 + 8 * th < clen) ? 48 + 8 * th : clen - 1); + raw_t3[0] = (float)g_raw[rb + (long long)t_c_0 * g_raw_row_stride]; + int t_c_1 = ((48 + 8 * th + 1 < clen) ? 48 + 8 * th + 1 : clen - 1); + raw_t3[1] = (float)g_raw[rb + (long long)t_c_1 * g_raw_row_stride]; + int t_c_2 = ((48 + 8 * th + 2 < clen) ? 48 + 8 * th + 2 : clen - 1); + raw_t3[2] = (float)g_raw[rb + (long long)t_c_2 * g_raw_row_stride]; + int t_c_3 = ((48 + 8 * th + 3 < clen) ? 48 + 8 * th + 3 : clen - 1); + raw_t3[3] = (float)g_raw[rb + (long long)t_c_3 * g_raw_row_stride]; + int t_c_4 = ((48 + 8 * th + 4 < clen) ? 48 + 8 * th + 4 : clen - 1); + raw_t3[4] = (float)g_raw[rb + (long long)t_c_4 * g_raw_row_stride]; + int t_c_5 = ((48 + 8 * th + 5 < clen) ? 48 + 8 * th + 5 : clen - 1); + raw_t3[5] = (float)g_raw[rb + (long long)t_c_5 * g_raw_row_stride]; + int t_c_6 = ((48 + 8 * th + 6 < clen) ? 48 + 8 * th + 6 : clen - 1); + raw_t3[6] = (float)g_raw[rb + (long long)t_c_6 * g_raw_row_stride]; + int t_c_7 = ((48 + 8 * th + 7 < clen) ? 48 + 8 * th + 7 : clen - 1); + raw_t3[7] = (float)g_raw[rb + (long long)t_c_7 * g_raw_row_stride]; + float raw_t2[8]; + long long rb_0 = ext0 * g_raw_row_stride + (long long)hv * g_raw_head_stride + (long long)d_lane; + int t_c_8 = ((32 + 8 * th < clen) ? 32 + 8 * th : clen - 1); + raw_t2[0] = (float)g_raw[rb_0 + (long long)t_c_8 * g_raw_row_stride]; + int t_c_9 = ((32 + 8 * th + 1 < clen) ? 32 + 8 * th + 1 : clen - 1); + raw_t2[1] = (float)g_raw[rb_0 + (long long)t_c_9 * g_raw_row_stride]; + int t_c_10 = ((32 + 8 * th + 2 < clen) ? 32 + 8 * th + 2 : clen - 1); + raw_t2[2] = (float)g_raw[rb_0 + (long long)t_c_10 * g_raw_row_stride]; + int t_c_11 = ((32 + 8 * th + 3 < clen) ? 32 + 8 * th + 3 : clen - 1); + raw_t2[3] = (float)g_raw[rb_0 + (long long)t_c_11 * g_raw_row_stride]; + int t_c_12 = ((32 + 8 * th + 4 < clen) ? 32 + 8 * th + 4 : clen - 1); + raw_t2[4] = (float)g_raw[rb_0 + (long long)t_c_12 * g_raw_row_stride]; + int t_c_13 = ((32 + 8 * th + 5 < clen) ? 32 + 8 * th + 5 : clen - 1); + raw_t2[5] = (float)g_raw[rb_0 + (long long)t_c_13 * g_raw_row_stride]; + int t_c_14 = ((32 + 8 * th + 6 < clen) ? 32 + 8 * th + 6 : clen - 1); + raw_t2[6] = (float)g_raw[rb_0 + (long long)t_c_14 * g_raw_row_stride]; + int t_c_15 = ((32 + 8 * th + 7 < clen) ? 32 + 8 * th + 7 : clen - 1); + raw_t2[7] = (float)g_raw[rb_0 + (long long)t_c_15 * g_raw_row_stride]; + { +#if __CUDA_ARCH__ == 1030 + uint32_t _addr_284 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_284), "f"(tot_0) : "memory"); +#else + uint32_t _addr_270 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_270), "f"(tot_0) : "memory"); +#endif + } + asm volatile("tcgen05.fence::before_thread_sync;"); + if (qd == 0) { + asm volatile("barrier.sync 2, 64;" ::: "memory"); + } + if (qd == 1) { + asm volatile("barrier.sync 3, 64;" ::: "memory"); + } + if (qd == 2) { + asm volatile("barrier.sync 4, 64;" ::: "memory"); + } + if (qd == 3) { + asm volatile("barrier.sync 5, 64;" ::: "memory"); + } asm volatile("tcgen05.fence::after_thread_sync;"); float t_other[1]; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&t_other[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + (1 - th) * 32 + lane_1)))); @@ -3836,132 +3865,280 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float dgr8[8]; float2 dyg2_0 = add_f32x2(make_float2(offset, offset), make_float2(sfx[0], sfx[1])); float2 rp_u = add_f32x2(make_float2(raw_t3[0], raw_t3[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_267 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_285 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_271 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_267), "l"(*(const unsigned long long*)&rp_u)); - float _abs_270; - asm("abs.f32 %0, %1;" : "=f"(_abs_270) : "f"(rp_z.x)); - float _neg_269; - asm("neg.f32 %0, %1;" : "=f"(_neg_269) : "f"(_abs_270)); - float _exp_268 = __nv_expf(_neg_269); - float _abs_273; - asm("abs.f32 %0, %1;" : "=f"(_abs_273) : "f"(rp_z.y)); - float _neg_272; - asm("neg.f32 %0, %1;" : "=f"(_neg_272) : "f"(_abs_273)); - float _exp_271 = __nv_expf(_neg_272); - float2 rp_e = make_float2(_exp_268, _exp_271); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_285), "l"(*(const unsigned long long*)&rp_u)); + float _abs_288; + asm("abs.f32 %0, %1;" : "=f"(_abs_288) : "f"(rp_z.x)); + float _neg_287; + asm("neg.f32 %0, %1;" : "=f"(_neg_287) : "f"(_abs_288)); + float _exp_286 = __nv_expf(_neg_287); + float _abs_291; + asm("abs.f32 %0, %1;" : "=f"(_abs_291) : "f"(rp_z.y)); + float _neg_290; + asm("neg.f32 %0, %1;" : "=f"(_neg_290) : "f"(_abs_291)); + float _exp_289 = __nv_expf(_neg_290); + float2 rp_e = make_float2(_exp_286, _exp_289); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z) : "l"(*(const unsigned long long*)&_f32x2_mul_a_271), "l"(*(const unsigned long long*)&rp_u)); + float _abs_274; + asm("abs.f32 %0, %1;" : "=f"(_abs_274) : "f"(rp_z.x)); + float _neg_273; + asm("neg.f32 %0, %1;" : "=f"(_neg_273) : "f"(_abs_274)); + float _exp_272 = __nv_expf(_neg_273); + float _abs_277; + asm("abs.f32 %0, %1;" : "=f"(_abs_277) : "f"(rp_z.y)); + float _neg_276; + asm("neg.f32 %0, %1;" : "=f"(_neg_276) : "f"(_abs_277)); + float _exp_275 = __nv_expf(_neg_276); + float2 rp_e = make_float2(_exp_272, _exp_275); +#endif float2 rp_dn = add_f32x2(make_float2(1.0f, 1.0f), rp_e); float2 rp_den; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den) : "l"(*(const unsigned long long*)&rp_dn), "l"(*(const unsigned long long*)&rp_dn)); - float _rcp_274 = approx_rcp(rp_den.x); - float _rcp_275 = approx_rcp(rp_den.y); - float2 _f32x2_mul_b_276 = make_float2(_rcp_274, _rcp_275); +#if __CUDA_ARCH__ == 1030 + float _rcp_292 = approx_rcp(rp_den.x); + float _rcp_293 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_294 = make_float2(_rcp_292, _rcp_293); +#else + float _rcp_278 = approx_rcp(rp_den.x); + float _rcp_279 = approx_rcp(rp_den.y); + float2 _f32x2_mul_b_280 = make_float2(_rcp_278, _rcp_279); +#endif float2 rp_sp; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_276)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_294)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp) : "l"(*(const unsigned long long*)&rp_e), "l"(*(const unsigned long long*)&_f32x2_mul_b_280)); +#endif float2 rp_t; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t) : "l"(*(const unsigned long long*)&dyg2_0), "l"(*(const unsigned long long*)&rp_sp)); - float2 _f32x2_mul_b_277 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_295 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_281 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_277)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_295)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr) : "l"(*(const unsigned long long*)&rp_t), "l"(*(const unsigned long long*)&_f32x2_mul_b_281)); +#endif u8[0] = rp_u.x; u8[1] = rp_u.y; dgr8[0] = rp_dgr.x; dgr8[1] = rp_dgr.y; float2 dyg2_1 = add_f32x2(make_float2(offset, offset), make_float2(sfx[2], sfx[3])); float2 rp_u_0 = add_f32x2(make_float2(raw_t3[2], raw_t3[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_278 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_296 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_282 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_278), "l"(*(const unsigned long long*)&rp_u_0)); - float _abs_281; - asm("abs.f32 %0, %1;" : "=f"(_abs_281) : "f"(rp_z_0.x)); - float _neg_280; - asm("neg.f32 %0, %1;" : "=f"(_neg_280) : "f"(_abs_281)); - float _exp_279 = __nv_expf(_neg_280); - float _abs_284; - asm("abs.f32 %0, %1;" : "=f"(_abs_284) : "f"(rp_z_0.y)); - float _neg_283; - asm("neg.f32 %0, %1;" : "=f"(_neg_283) : "f"(_abs_284)); - float _exp_282 = __nv_expf(_neg_283); - float2 rp_e_0 = make_float2(_exp_279, _exp_282); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_296), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_299; + asm("abs.f32 %0, %1;" : "=f"(_abs_299) : "f"(rp_z_0.x)); + float _neg_298; + asm("neg.f32 %0, %1;" : "=f"(_neg_298) : "f"(_abs_299)); + float _exp_297 = __nv_expf(_neg_298); + float _abs_302; + asm("abs.f32 %0, %1;" : "=f"(_abs_302) : "f"(rp_z_0.y)); + float _neg_301; + asm("neg.f32 %0, %1;" : "=f"(_neg_301) : "f"(_abs_302)); + float _exp_300 = __nv_expf(_neg_301); + float2 rp_e_0 = make_float2(_exp_297, _exp_300); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_282), "l"(*(const unsigned long long*)&rp_u_0)); + float _abs_285; + asm("abs.f32 %0, %1;" : "=f"(_abs_285) : "f"(rp_z_0.x)); + float _neg_284; + asm("neg.f32 %0, %1;" : "=f"(_neg_284) : "f"(_abs_285)); + float _exp_283 = __nv_expf(_neg_284); + float _abs_288; + asm("abs.f32 %0, %1;" : "=f"(_abs_288) : "f"(rp_z_0.y)); + float _neg_287; + asm("neg.f32 %0, %1;" : "=f"(_neg_287) : "f"(_abs_288)); + float _exp_286 = __nv_expf(_neg_287); + float2 rp_e_0 = make_float2(_exp_283, _exp_286); +#endif float2 rp_dn_0 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_0); float2 rp_den_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_0) : "l"(*(const unsigned long long*)&rp_dn_0), "l"(*(const unsigned long long*)&rp_dn_0)); - float _rcp_285 = approx_rcp(rp_den_0.x); - float _rcp_286 = approx_rcp(rp_den_0.y); - float2 _f32x2_mul_b_287 = make_float2(_rcp_285, _rcp_286); +#if __CUDA_ARCH__ == 1030 + float _rcp_303 = approx_rcp(rp_den_0.x); + float _rcp_304 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_305 = make_float2(_rcp_303, _rcp_304); +#else + float _rcp_289 = approx_rcp(rp_den_0.x); + float _rcp_290 = approx_rcp(rp_den_0.y); + float2 _f32x2_mul_b_291 = make_float2(_rcp_289, _rcp_290); +#endif float2 rp_sp_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_287)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_305)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_0) : "l"(*(const unsigned long long*)&rp_e_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_291)); +#endif float2 rp_t_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_0) : "l"(*(const unsigned long long*)&dyg2_1), "l"(*(const unsigned long long*)&rp_sp_0)); - float2 _f32x2_mul_b_288 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_306 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_292 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_0; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_288)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_306)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_0) : "l"(*(const unsigned long long*)&rp_t_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_292)); +#endif u8[2] = rp_u_0.x; u8[3] = rp_u_0.y; dgr8[2] = rp_dgr_0.x; dgr8[3] = rp_dgr_0.y; float2 dyg2_2 = add_f32x2(make_float2(offset, offset), make_float2(sfx[4], sfx[5])); float2 rp_u_1 = add_f32x2(make_float2(raw_t3[4], raw_t3[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_289 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_307 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_293 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_289), "l"(*(const unsigned long long*)&rp_u_1)); - float _abs_292; - asm("abs.f32 %0, %1;" : "=f"(_abs_292) : "f"(rp_z_1.x)); - float _neg_291; - asm("neg.f32 %0, %1;" : "=f"(_neg_291) : "f"(_abs_292)); - float _exp_290 = __nv_expf(_neg_291); - float _abs_295; - asm("abs.f32 %0, %1;" : "=f"(_abs_295) : "f"(rp_z_1.y)); - float _neg_294; - asm("neg.f32 %0, %1;" : "=f"(_neg_294) : "f"(_abs_295)); - float _exp_293 = __nv_expf(_neg_294); - float2 rp_e_1 = make_float2(_exp_290, _exp_293); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_307), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_310; + asm("abs.f32 %0, %1;" : "=f"(_abs_310) : "f"(rp_z_1.x)); + float _neg_309; + asm("neg.f32 %0, %1;" : "=f"(_neg_309) : "f"(_abs_310)); + float _exp_308 = __nv_expf(_neg_309); + float _abs_313; + asm("abs.f32 %0, %1;" : "=f"(_abs_313) : "f"(rp_z_1.y)); + float _neg_312; + asm("neg.f32 %0, %1;" : "=f"(_neg_312) : "f"(_abs_313)); + float _exp_311 = __nv_expf(_neg_312); + float2 rp_e_1 = make_float2(_exp_308, _exp_311); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_293), "l"(*(const unsigned long long*)&rp_u_1)); + float _abs_296; + asm("abs.f32 %0, %1;" : "=f"(_abs_296) : "f"(rp_z_1.x)); + float _neg_295; + asm("neg.f32 %0, %1;" : "=f"(_neg_295) : "f"(_abs_296)); + float _exp_294 = __nv_expf(_neg_295); + float _abs_299; + asm("abs.f32 %0, %1;" : "=f"(_abs_299) : "f"(rp_z_1.y)); + float _neg_298; + asm("neg.f32 %0, %1;" : "=f"(_neg_298) : "f"(_abs_299)); + float _exp_297 = __nv_expf(_neg_298); + float2 rp_e_1 = make_float2(_exp_294, _exp_297); +#endif float2 rp_dn_1 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_1); float2 rp_den_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_1) : "l"(*(const unsigned long long*)&rp_dn_1), "l"(*(const unsigned long long*)&rp_dn_1)); - float _rcp_296 = approx_rcp(rp_den_1.x); - float _rcp_297 = approx_rcp(rp_den_1.y); - float2 _f32x2_mul_b_298 = make_float2(_rcp_296, _rcp_297); +#if __CUDA_ARCH__ == 1030 + float _rcp_314 = approx_rcp(rp_den_1.x); + float _rcp_315 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_316 = make_float2(_rcp_314, _rcp_315); +#else + float _rcp_300 = approx_rcp(rp_den_1.x); + float _rcp_301 = approx_rcp(rp_den_1.y); + float2 _f32x2_mul_b_302 = make_float2(_rcp_300, _rcp_301); +#endif float2 rp_sp_1; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_298)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_316)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_1) : "l"(*(const unsigned long long*)&rp_e_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_302)); +#endif float2 rp_t_1; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_1) : "l"(*(const unsigned long long*)&dyg2_2), "l"(*(const unsigned long long*)&rp_sp_1)); - float2 _f32x2_mul_b_299 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_317 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_303 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_299)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_317)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_1) : "l"(*(const unsigned long long*)&rp_t_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_303)); +#endif u8[4] = rp_u_1.x; u8[5] = rp_u_1.y; dgr8[4] = rp_dgr_1.x; dgr8[5] = rp_dgr_1.y; float2 dyg2_3 = add_f32x2(make_float2(offset, offset), make_float2(sfx[6], sfx[7])); float2 rp_u_2 = add_f32x2(make_float2(raw_t3[6], raw_t3[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_300 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_318 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_304 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_300), "l"(*(const unsigned long long*)&rp_u_2)); - float _abs_303; - asm("abs.f32 %0, %1;" : "=f"(_abs_303) : "f"(rp_z_2.x)); - float _neg_302; - asm("neg.f32 %0, %1;" : "=f"(_neg_302) : "f"(_abs_303)); - float _exp_301 = __nv_expf(_neg_302); - float _abs_306; - asm("abs.f32 %0, %1;" : "=f"(_abs_306) : "f"(rp_z_2.y)); - float _neg_305; - asm("neg.f32 %0, %1;" : "=f"(_neg_305) : "f"(_abs_306)); - float _exp_304 = __nv_expf(_neg_305); - float2 rp_e_2 = make_float2(_exp_301, _exp_304); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_318), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_321; + asm("abs.f32 %0, %1;" : "=f"(_abs_321) : "f"(rp_z_2.x)); + float _neg_320; + asm("neg.f32 %0, %1;" : "=f"(_neg_320) : "f"(_abs_321)); + float _exp_319 = __nv_expf(_neg_320); + float _abs_324; + asm("abs.f32 %0, %1;" : "=f"(_abs_324) : "f"(rp_z_2.y)); + float _neg_323; + asm("neg.f32 %0, %1;" : "=f"(_neg_323) : "f"(_abs_324)); + float _exp_322 = __nv_expf(_neg_323); + float2 rp_e_2 = make_float2(_exp_319, _exp_322); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_304), "l"(*(const unsigned long long*)&rp_u_2)); + float _abs_307; + asm("abs.f32 %0, %1;" : "=f"(_abs_307) : "f"(rp_z_2.x)); + float _neg_306; + asm("neg.f32 %0, %1;" : "=f"(_neg_306) : "f"(_abs_307)); + float _exp_305 = __nv_expf(_neg_306); + float _abs_310; + asm("abs.f32 %0, %1;" : "=f"(_abs_310) : "f"(rp_z_2.y)); + float _neg_309; + asm("neg.f32 %0, %1;" : "=f"(_neg_309) : "f"(_abs_310)); + float _exp_308 = __nv_expf(_neg_309); + float2 rp_e_2 = make_float2(_exp_305, _exp_308); +#endif float2 rp_dn_2 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_2); float2 rp_den_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_2) : "l"(*(const unsigned long long*)&rp_dn_2), "l"(*(const unsigned long long*)&rp_dn_2)); - float _rcp_307 = approx_rcp(rp_den_2.x); - float _rcp_308 = approx_rcp(rp_den_2.y); - float2 _f32x2_mul_b_309 = make_float2(_rcp_307, _rcp_308); +#if __CUDA_ARCH__ == 1030 + float _rcp_325 = approx_rcp(rp_den_2.x); + float _rcp_326 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_327 = make_float2(_rcp_325, _rcp_326); +#else + float _rcp_311 = approx_rcp(rp_den_2.x); + float _rcp_312 = approx_rcp(rp_den_2.y); + float2 _f32x2_mul_b_313 = make_float2(_rcp_311, _rcp_312); +#endif float2 rp_sp_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_309)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_327)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_2) : "l"(*(const unsigned long long*)&rp_e_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_313)); +#endif float2 rp_t_2; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_2) : "l"(*(const unsigned long long*)&dyg2_3), "l"(*(const unsigned long long*)&rp_sp_2)); - float2 _f32x2_mul_b_310 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_328 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_314 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_310)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_328)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_2) : "l"(*(const unsigned long long*)&rp_t_2), "l"(*(const unsigned long long*)&_f32x2_mul_b_314)); +#endif u8[6] = rp_u_2.x; u8[7] = rp_u_2.y; dgr8[6] = rp_dgr_2.x; @@ -3976,54 +4153,102 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d } unsigned int pv_0 = __shfl_xor_sync(0xFFFFFFFF, pw[0], 1); unsigned int pq_0[1]; - uint32_t _prmt_311; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_311) : "r"(pw[0]), "r"(pv_0), "r"(sel)); - pq_0[0] = _prmt_311; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_329; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_329) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_329; +#else + uint32_t _prmt_315; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[0]), "r"(pv_0), "r"(sel)); + pq_0[0] = _prmt_315; +#endif int t_q_0 = 48 + 8 * th + odd; if (t_q_0 < clen) { { - auto _gst_312 = draw_base + (t_q_0 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = pq_0[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_330 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast(_gst_330) + (0)) = pq_0[0]; +#else + auto _gst_316 = draw_base + (t_q_0 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast(_gst_316) + (0)) = pq_0[0]; +#endif } } unsigned int pv_1 = __shfl_xor_sync(0xFFFFFFFF, pw[1], 1); unsigned int pq_1[1]; - uint32_t _prmt_313; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_313) : "r"(pw[1]), "r"(pv_1), "r"(sel)); - pq_1[0] = _prmt_313; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_331; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_331) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_331; +#else + uint32_t _prmt_317; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[1]), "r"(pv_1), "r"(sel)); + pq_1[0] = _prmt_317; +#endif int t_q_1 = 48 + 8 * th + 2 + odd; if (t_q_1 < clen) { { - auto _gst_314 = draw_base + (t_q_1 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = pq_1[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_332 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_332)); + *(reinterpret_cast(_gst_332) + (0)) = pq_1[0]; +#else + auto _gst_318 = draw_base + (t_q_1 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast(_gst_318) + (0)) = pq_1[0]; +#endif } } unsigned int pv_2 = __shfl_xor_sync(0xFFFFFFFF, pw[2], 1); unsigned int pq_2[1]; - uint32_t _prmt_315; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_315) : "r"(pw[2]), "r"(pv_2), "r"(sel)); - pq_2[0] = _prmt_315; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_333; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_333) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_333; +#else + uint32_t _prmt_319; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_319) : "r"(pw[2]), "r"(pv_2), "r"(sel)); + pq_2[0] = _prmt_319; +#endif int t_q_2 = 48 + 8 * th + 4 + odd; if (t_q_2 < clen) { { - auto _gst_316 = draw_base + (t_q_2 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = pq_2[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_334 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_334)); + *(reinterpret_cast(_gst_334) + (0)) = pq_2[0]; +#else + auto _gst_320 = draw_base + (t_q_2 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = pq_2[0]; +#endif } } unsigned int pv_3 = __shfl_xor_sync(0xFFFFFFFF, pw[3], 1); unsigned int pq_3[1]; - uint32_t _prmt_317; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_317) : "r"(pw[3]), "r"(pv_3), "r"(sel)); - pq_3[0] = _prmt_317; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_335; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_335) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_335; +#else + uint32_t _prmt_321; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_321) : "r"(pw[3]), "r"(pv_3), "r"(sel)); + pq_3[0] = _prmt_321; +#endif int t_q_3 = 48 + 8 * th + 6 + odd; if (t_q_3 < clen) { { - auto _gst_318 = draw_base + (t_q_3 * rs_g - odd); - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = pq_3[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_336 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_336)); + *(reinterpret_cast(_gst_336) + (0)) = pq_3[0]; +#else + auto _gst_322 = draw_base + (t_q_3 * rs_g - odd); + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast(_gst_322) + (0)) = pq_3[0]; +#endif } } int t_a_0 = 48 + 8 * th; @@ -4088,47 +4313,99 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa) : "f"(pa01), "f"(pa23)); float nm_t; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t) : "f"(s_a), "f"(pa)); - float _sub_319; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_319) : "f"(s_a), "f"(nm_t)); +#if __CUDA_ARCH__ == 1030 + float _sub_337; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_337) : "f"(s_a), "f"(nm_t)); +#else + float _sub_323; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_323) : "f"(s_a), "f"(nm_t)); +#endif float nm_es; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_319), "f"(pa)); - float _sub_320; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_320) : "f"(pa), "f"(nm_t)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_337), "f"(pa)); + float _sub_338; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_338) : "f"(pa), "f"(nm_t)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es) : "f"(_sub_323), "f"(pa)); + float _sub_324; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_324) : "f"(pa), "f"(nm_t)); +#endif float nm_ex; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_320), "f"(s_a)); - float _abs_322; - asm("abs.f32 %0, %1;" : "=f"(_abs_322) : "f"(s_a)); - float _abs_323; - asm("abs.f32 %0, %1;" : "=f"(_abs_323) : "f"(pa)); - uint32_t _setp_321_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_321_bits) : "f"(_abs_322), "f"(_abs_323)); - bool _setp_321 = _setp_321_bits != 0; - float nm_err = ((_setp_321) ? nm_es : nm_ex); - float _add_324; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_324) : "f"(c_a), "f"(nm_err)); - c_a = _add_324; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_338), "f"(s_a)); + float _abs_340; + asm("abs.f32 %0, %1;" : "=f"(_abs_340) : "f"(s_a)); + float _abs_341; + asm("abs.f32 %0, %1;" : "=f"(_abs_341) : "f"(pa)); + uint32_t _setp_339_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_339_bits) : "f"(_abs_340), "f"(_abs_341)); + bool _setp_339 = _setp_339_bits != 0; + float nm_err = ((_setp_339) ? nm_es : nm_ex); + float _add_342; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_342) : "f"(c_a), "f"(nm_err)); + c_a = _add_342; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex) : "f"(_sub_324), "f"(s_a)); + float _abs_326; + asm("abs.f32 %0, %1;" : "=f"(_abs_326) : "f"(s_a)); + float _abs_327; + asm("abs.f32 %0, %1;" : "=f"(_abs_327) : "f"(pa)); + uint32_t _setp_325_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_325_bits) : "f"(_abs_326), "f"(_abs_327)); + bool _setp_325 = _setp_325_bits != 0; + float nm_err = ((_setp_325) ? nm_es : nm_ex); + float _add_328; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(c_a), "f"(nm_err)); + c_a = _add_328; +#endif s_a = nm_t; float nm_t_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_0) : "f"(s_b), "f"(pb)); - float _sub_325; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_325) : "f"(s_b), "f"(nm_t_0)); +#if __CUDA_ARCH__ == 1030 + float _sub_343; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_343) : "f"(s_b), "f"(nm_t_0)); +#else + float _sub_329; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_329) : "f"(s_b), "f"(nm_t_0)); +#endif float nm_es_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_325), "f"(pb)); - float _sub_326; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_326) : "f"(pb), "f"(nm_t_0)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_343), "f"(pb)); + float _sub_344; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_344) : "f"(pb), "f"(nm_t_0)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_0) : "f"(_sub_329), "f"(pb)); + float _sub_330; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_330) : "f"(pb), "f"(nm_t_0)); +#endif float nm_ex_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_326), "f"(s_b)); - float _abs_328; - asm("abs.f32 %0, %1;" : "=f"(_abs_328) : "f"(s_b)); - float _abs_329; - asm("abs.f32 %0, %1;" : "=f"(_abs_329) : "f"(pb)); - uint32_t _setp_327_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_327_bits) : "f"(_abs_328), "f"(_abs_329)); - bool _setp_327 = _setp_327_bits != 0; - float nm_err_0 = ((_setp_327) ? nm_es_0 : nm_ex_0); - float _add_330; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_330) : "f"(c_b), "f"(nm_err_0)); - c_b = _add_330; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_344), "f"(s_b)); + float _abs_346; + asm("abs.f32 %0, %1;" : "=f"(_abs_346) : "f"(s_b)); + float _abs_347; + asm("abs.f32 %0, %1;" : "=f"(_abs_347) : "f"(pb)); + uint32_t _setp_345_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_345_bits) : "f"(_abs_346), "f"(_abs_347)); + bool _setp_345 = _setp_345_bits != 0; + float nm_err_0 = ((_setp_345) ? nm_es_0 : nm_ex_0); + float _add_348; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_348) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_348; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_0) : "f"(_sub_330), "f"(s_b)); + float _abs_332; + asm("abs.f32 %0, %1;" : "=f"(_abs_332) : "f"(s_b)); + float _abs_333; + asm("abs.f32 %0, %1;" : "=f"(_abs_333) : "f"(pb)); + uint32_t _setp_331_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_331_bits) : "f"(_abs_332), "f"(_abs_333)); + bool _setp_331 = _setp_331_bits != 0; + float nm_err_0 = ((_setp_331) ? nm_es_0 : nm_ex_0); + float _add_334; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_334) : "f"(c_b), "f"(nm_err_0)); + c_b = _add_334; +#endif s_b = nm_t_0; gacc[0] = s_a; gacc[1] = c_a; @@ -4152,132 +4429,280 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float dgr8_0[8]; float2 dyg2_4 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[0], sfx_0[1])); float2 rp_u_3 = add_f32x2(make_float2(raw_t2[0], raw_t2[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_331 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_349 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_335 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_331), "l"(*(const unsigned long long*)&rp_u_3)); - float _abs_334; - asm("abs.f32 %0, %1;" : "=f"(_abs_334) : "f"(rp_z_3.x)); - float _neg_333; - asm("neg.f32 %0, %1;" : "=f"(_neg_333) : "f"(_abs_334)); - float _exp_332 = __nv_expf(_neg_333); - float _abs_337; - asm("abs.f32 %0, %1;" : "=f"(_abs_337) : "f"(rp_z_3.y)); - float _neg_336; - asm("neg.f32 %0, %1;" : "=f"(_neg_336) : "f"(_abs_337)); - float _exp_335 = __nv_expf(_neg_336); - float2 rp_e_3 = make_float2(_exp_332, _exp_335); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_349), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_352; + asm("abs.f32 %0, %1;" : "=f"(_abs_352) : "f"(rp_z_3.x)); + float _neg_351; + asm("neg.f32 %0, %1;" : "=f"(_neg_351) : "f"(_abs_352)); + float _exp_350 = __nv_expf(_neg_351); + float _abs_355; + asm("abs.f32 %0, %1;" : "=f"(_abs_355) : "f"(rp_z_3.y)); + float _neg_354; + asm("neg.f32 %0, %1;" : "=f"(_neg_354) : "f"(_abs_355)); + float _exp_353 = __nv_expf(_neg_354); + float2 rp_e_3 = make_float2(_exp_350, _exp_353); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_335), "l"(*(const unsigned long long*)&rp_u_3)); + float _abs_338; + asm("abs.f32 %0, %1;" : "=f"(_abs_338) : "f"(rp_z_3.x)); + float _neg_337; + asm("neg.f32 %0, %1;" : "=f"(_neg_337) : "f"(_abs_338)); + float _exp_336 = __nv_expf(_neg_337); + float _abs_341; + asm("abs.f32 %0, %1;" : "=f"(_abs_341) : "f"(rp_z_3.y)); + float _neg_340; + asm("neg.f32 %0, %1;" : "=f"(_neg_340) : "f"(_abs_341)); + float _exp_339 = __nv_expf(_neg_340); + float2 rp_e_3 = make_float2(_exp_336, _exp_339); +#endif float2 rp_dn_3 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_3); float2 rp_den_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_3) : "l"(*(const unsigned long long*)&rp_dn_3), "l"(*(const unsigned long long*)&rp_dn_3)); - float _rcp_338 = approx_rcp(rp_den_3.x); - float _rcp_339 = approx_rcp(rp_den_3.y); - float2 _f32x2_mul_b_340 = make_float2(_rcp_338, _rcp_339); +#if __CUDA_ARCH__ == 1030 + float _rcp_356 = approx_rcp(rp_den_3.x); + float _rcp_357 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_358 = make_float2(_rcp_356, _rcp_357); +#else + float _rcp_342 = approx_rcp(rp_den_3.x); + float _rcp_343 = approx_rcp(rp_den_3.y); + float2 _f32x2_mul_b_344 = make_float2(_rcp_342, _rcp_343); +#endif float2 rp_sp_3; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_340)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_358)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_3) : "l"(*(const unsigned long long*)&rp_e_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_344)); +#endif float2 rp_t_3; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_3) : "l"(*(const unsigned long long*)&dyg2_4), "l"(*(const unsigned long long*)&rp_sp_3)); - float2 _f32x2_mul_b_341 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_359 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_345 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_341)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_359)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_3) : "l"(*(const unsigned long long*)&rp_t_3), "l"(*(const unsigned long long*)&_f32x2_mul_b_345)); +#endif u8_0[0] = rp_u_3.x; u8_0[1] = rp_u_3.y; dgr8_0[0] = rp_dgr_3.x; dgr8_0[1] = rp_dgr_3.y; float2 dyg2_5 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[2], sfx_0[3])); float2 rp_u_4 = add_f32x2(make_float2(raw_t2[2], raw_t2[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_342 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_360 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_346 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_342), "l"(*(const unsigned long long*)&rp_u_4)); - float _abs_345; - asm("abs.f32 %0, %1;" : "=f"(_abs_345) : "f"(rp_z_4.x)); - float _neg_344; - asm("neg.f32 %0, %1;" : "=f"(_neg_344) : "f"(_abs_345)); - float _exp_343 = __nv_expf(_neg_344); - float _abs_348; - asm("abs.f32 %0, %1;" : "=f"(_abs_348) : "f"(rp_z_4.y)); - float _neg_347; - asm("neg.f32 %0, %1;" : "=f"(_neg_347) : "f"(_abs_348)); - float _exp_346 = __nv_expf(_neg_347); - float2 rp_e_4 = make_float2(_exp_343, _exp_346); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_360), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_363; + asm("abs.f32 %0, %1;" : "=f"(_abs_363) : "f"(rp_z_4.x)); + float _neg_362; + asm("neg.f32 %0, %1;" : "=f"(_neg_362) : "f"(_abs_363)); + float _exp_361 = __nv_expf(_neg_362); + float _abs_366; + asm("abs.f32 %0, %1;" : "=f"(_abs_366) : "f"(rp_z_4.y)); + float _neg_365; + asm("neg.f32 %0, %1;" : "=f"(_neg_365) : "f"(_abs_366)); + float _exp_364 = __nv_expf(_neg_365); + float2 rp_e_4 = make_float2(_exp_361, _exp_364); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_346), "l"(*(const unsigned long long*)&rp_u_4)); + float _abs_349; + asm("abs.f32 %0, %1;" : "=f"(_abs_349) : "f"(rp_z_4.x)); + float _neg_348; + asm("neg.f32 %0, %1;" : "=f"(_neg_348) : "f"(_abs_349)); + float _exp_347 = __nv_expf(_neg_348); + float _abs_352; + asm("abs.f32 %0, %1;" : "=f"(_abs_352) : "f"(rp_z_4.y)); + float _neg_351; + asm("neg.f32 %0, %1;" : "=f"(_neg_351) : "f"(_abs_352)); + float _exp_350 = __nv_expf(_neg_351); + float2 rp_e_4 = make_float2(_exp_347, _exp_350); +#endif float2 rp_dn_4 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_4); float2 rp_den_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_4) : "l"(*(const unsigned long long*)&rp_dn_4), "l"(*(const unsigned long long*)&rp_dn_4)); - float _rcp_349 = approx_rcp(rp_den_4.x); - float _rcp_350 = approx_rcp(rp_den_4.y); - float2 _f32x2_mul_b_351 = make_float2(_rcp_349, _rcp_350); +#if __CUDA_ARCH__ == 1030 + float _rcp_367 = approx_rcp(rp_den_4.x); + float _rcp_368 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_369 = make_float2(_rcp_367, _rcp_368); +#else + float _rcp_353 = approx_rcp(rp_den_4.x); + float _rcp_354 = approx_rcp(rp_den_4.y); + float2 _f32x2_mul_b_355 = make_float2(_rcp_353, _rcp_354); +#endif float2 rp_sp_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_351)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_369)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_4) : "l"(*(const unsigned long long*)&rp_e_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_355)); +#endif float2 rp_t_4; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_4) : "l"(*(const unsigned long long*)&dyg2_5), "l"(*(const unsigned long long*)&rp_sp_4)); - float2 _f32x2_mul_b_352 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_370 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_356 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_352)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_370)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_4) : "l"(*(const unsigned long long*)&rp_t_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_356)); +#endif u8_0[2] = rp_u_4.x; u8_0[3] = rp_u_4.y; dgr8_0[2] = rp_dgr_4.x; dgr8_0[3] = rp_dgr_4.y; float2 dyg2_6 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[4], sfx_0[5])); float2 rp_u_5 = add_f32x2(make_float2(raw_t2[4], raw_t2[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_353 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_371 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_357 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_353), "l"(*(const unsigned long long*)&rp_u_5)); - float _abs_356; - asm("abs.f32 %0, %1;" : "=f"(_abs_356) : "f"(rp_z_5.x)); - float _neg_355; - asm("neg.f32 %0, %1;" : "=f"(_neg_355) : "f"(_abs_356)); - float _exp_354 = __nv_expf(_neg_355); - float _abs_359; - asm("abs.f32 %0, %1;" : "=f"(_abs_359) : "f"(rp_z_5.y)); - float _neg_358; - asm("neg.f32 %0, %1;" : "=f"(_neg_358) : "f"(_abs_359)); - float _exp_357 = __nv_expf(_neg_358); - float2 rp_e_5 = make_float2(_exp_354, _exp_357); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_371), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_374; + asm("abs.f32 %0, %1;" : "=f"(_abs_374) : "f"(rp_z_5.x)); + float _neg_373; + asm("neg.f32 %0, %1;" : "=f"(_neg_373) : "f"(_abs_374)); + float _exp_372 = __nv_expf(_neg_373); + float _abs_377; + asm("abs.f32 %0, %1;" : "=f"(_abs_377) : "f"(rp_z_5.y)); + float _neg_376; + asm("neg.f32 %0, %1;" : "=f"(_neg_376) : "f"(_abs_377)); + float _exp_375 = __nv_expf(_neg_376); + float2 rp_e_5 = make_float2(_exp_372, _exp_375); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_357), "l"(*(const unsigned long long*)&rp_u_5)); + float _abs_360; + asm("abs.f32 %0, %1;" : "=f"(_abs_360) : "f"(rp_z_5.x)); + float _neg_359; + asm("neg.f32 %0, %1;" : "=f"(_neg_359) : "f"(_abs_360)); + float _exp_358 = __nv_expf(_neg_359); + float _abs_363; + asm("abs.f32 %0, %1;" : "=f"(_abs_363) : "f"(rp_z_5.y)); + float _neg_362; + asm("neg.f32 %0, %1;" : "=f"(_neg_362) : "f"(_abs_363)); + float _exp_361 = __nv_expf(_neg_362); + float2 rp_e_5 = make_float2(_exp_358, _exp_361); +#endif float2 rp_dn_5 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_5); float2 rp_den_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_5) : "l"(*(const unsigned long long*)&rp_dn_5), "l"(*(const unsigned long long*)&rp_dn_5)); - float _rcp_360 = approx_rcp(rp_den_5.x); - float _rcp_361 = approx_rcp(rp_den_5.y); - float2 _f32x2_mul_b_362 = make_float2(_rcp_360, _rcp_361); +#if __CUDA_ARCH__ == 1030 + float _rcp_378 = approx_rcp(rp_den_5.x); + float _rcp_379 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_380 = make_float2(_rcp_378, _rcp_379); +#else + float _rcp_364 = approx_rcp(rp_den_5.x); + float _rcp_365 = approx_rcp(rp_den_5.y); + float2 _f32x2_mul_b_366 = make_float2(_rcp_364, _rcp_365); +#endif float2 rp_sp_5; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_362)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_380)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_5) : "l"(*(const unsigned long long*)&rp_e_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_366)); +#endif float2 rp_t_5; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_5) : "l"(*(const unsigned long long*)&dyg2_6), "l"(*(const unsigned long long*)&rp_sp_5)); - float2 _f32x2_mul_b_363 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_381 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_367 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_363)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_381)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_5) : "l"(*(const unsigned long long*)&rp_t_5), "l"(*(const unsigned long long*)&_f32x2_mul_b_367)); +#endif u8_0[4] = rp_u_5.x; u8_0[5] = rp_u_5.y; dgr8_0[4] = rp_dgr_5.x; dgr8_0[5] = rp_dgr_5.y; float2 dyg2_7 = add_f32x2(make_float2(offset_0, offset_0), make_float2(sfx_0[6], sfx_0[7])); float2 rp_u_6 = add_f32x2(make_float2(raw_t2[6], raw_t2[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_364 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_382 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_368 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_364), "l"(*(const unsigned long long*)&rp_u_6)); - float _abs_367; - asm("abs.f32 %0, %1;" : "=f"(_abs_367) : "f"(rp_z_6.x)); - float _neg_366; - asm("neg.f32 %0, %1;" : "=f"(_neg_366) : "f"(_abs_367)); - float _exp_365 = __nv_expf(_neg_366); - float _abs_370; - asm("abs.f32 %0, %1;" : "=f"(_abs_370) : "f"(rp_z_6.y)); - float _neg_369; - asm("neg.f32 %0, %1;" : "=f"(_neg_369) : "f"(_abs_370)); - float _exp_368 = __nv_expf(_neg_369); - float2 rp_e_6 = make_float2(_exp_365, _exp_368); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_382), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_385; + asm("abs.f32 %0, %1;" : "=f"(_abs_385) : "f"(rp_z_6.x)); + float _neg_384; + asm("neg.f32 %0, %1;" : "=f"(_neg_384) : "f"(_abs_385)); + float _exp_383 = __nv_expf(_neg_384); + float _abs_388; + asm("abs.f32 %0, %1;" : "=f"(_abs_388) : "f"(rp_z_6.y)); + float _neg_387; + asm("neg.f32 %0, %1;" : "=f"(_neg_387) : "f"(_abs_388)); + float _exp_386 = __nv_expf(_neg_387); + float2 rp_e_6 = make_float2(_exp_383, _exp_386); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_368), "l"(*(const unsigned long long*)&rp_u_6)); + float _abs_371; + asm("abs.f32 %0, %1;" : "=f"(_abs_371) : "f"(rp_z_6.x)); + float _neg_370; + asm("neg.f32 %0, %1;" : "=f"(_neg_370) : "f"(_abs_371)); + float _exp_369 = __nv_expf(_neg_370); + float _abs_374; + asm("abs.f32 %0, %1;" : "=f"(_abs_374) : "f"(rp_z_6.y)); + float _neg_373; + asm("neg.f32 %0, %1;" : "=f"(_neg_373) : "f"(_abs_374)); + float _exp_372 = __nv_expf(_neg_373); + float2 rp_e_6 = make_float2(_exp_369, _exp_372); +#endif float2 rp_dn_6 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_6); float2 rp_den_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_6) : "l"(*(const unsigned long long*)&rp_dn_6), "l"(*(const unsigned long long*)&rp_dn_6)); - float _rcp_371 = approx_rcp(rp_den_6.x); - float _rcp_372 = approx_rcp(rp_den_6.y); - float2 _f32x2_mul_b_373 = make_float2(_rcp_371, _rcp_372); +#if __CUDA_ARCH__ == 1030 + float _rcp_389 = approx_rcp(rp_den_6.x); + float _rcp_390 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_391 = make_float2(_rcp_389, _rcp_390); +#else + float _rcp_375 = approx_rcp(rp_den_6.x); + float _rcp_376 = approx_rcp(rp_den_6.y); + float2 _f32x2_mul_b_377 = make_float2(_rcp_375, _rcp_376); +#endif float2 rp_sp_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_373)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_391)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_6) : "l"(*(const unsigned long long*)&rp_e_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_377)); +#endif float2 rp_t_6; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_6) : "l"(*(const unsigned long long*)&dyg2_7), "l"(*(const unsigned long long*)&rp_sp_6)); - float2 _f32x2_mul_b_374 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_392 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_378 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_374)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_392)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_6) : "l"(*(const unsigned long long*)&rp_t_6), "l"(*(const unsigned long long*)&_f32x2_mul_b_378)); +#endif u8_0[6] = rp_u_6.x; u8_0[7] = rp_u_6.y; dgr8_0[6] = rp_dgr_6.x; @@ -4292,54 +4717,102 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d } unsigned int pv_4 = __shfl_xor_sync(0xFFFFFFFF, pw_0[0], 1); unsigned int pq_4[1]; - uint32_t _prmt_375; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_375) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); - pq_4[0] = _prmt_375; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_393; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_393) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_393; +#else + uint32_t _prmt_379; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[0]), "r"(pv_4), "r"(sel_0)); + pq_4[0] = _prmt_379; +#endif int t_q_4 = 32 + 8 * th + odd_0; if (t_q_4 < clen) { { - auto _gst_376 = draw_base + (t_q_4 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_376)); - *(reinterpret_cast(_gst_376) + (0)) = pq_4[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_394 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_394)); + *(reinterpret_cast(_gst_394) + (0)) = pq_4[0]; +#else + auto _gst_380 = draw_base + (t_q_4 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = pq_4[0]; +#endif } } unsigned int pv_5 = __shfl_xor_sync(0xFFFFFFFF, pw_0[1], 1); unsigned int pq_5[1]; - uint32_t _prmt_377; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_377) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); - pq_5[0] = _prmt_377; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_395; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_395) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_395; +#else + uint32_t _prmt_381; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[1]), "r"(pv_5), "r"(sel_0)); + pq_5[0] = _prmt_381; +#endif int t_q_5 = 32 + 8 * th + 2 + odd_0; if (t_q_5 < clen) { { - auto _gst_378 = draw_base + (t_q_5 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_378)); - *(reinterpret_cast(_gst_378) + (0)) = pq_5[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_396 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_396)); + *(reinterpret_cast(_gst_396) + (0)) = pq_5[0]; +#else + auto _gst_382 = draw_base + (t_q_5 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = pq_5[0]; +#endif } } unsigned int pv_6 = __shfl_xor_sync(0xFFFFFFFF, pw_0[2], 1); unsigned int pq_6[1]; - uint32_t _prmt_379; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_379) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); - pq_6[0] = _prmt_379; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_397; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_397) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_397; +#else + uint32_t _prmt_383; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_383) : "r"(pw_0[2]), "r"(pv_6), "r"(sel_0)); + pq_6[0] = _prmt_383; +#endif int t_q_6 = 32 + 8 * th + 4 + odd_0; if (t_q_6 < clen) { { - auto _gst_380 = draw_base + (t_q_6 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_380)); - *(reinterpret_cast(_gst_380) + (0)) = pq_6[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_398 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_398)); + *(reinterpret_cast(_gst_398) + (0)) = pq_6[0]; +#else + auto _gst_384 = draw_base + (t_q_6 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = pq_6[0]; +#endif } } unsigned int pv_7 = __shfl_xor_sync(0xFFFFFFFF, pw_0[3], 1); unsigned int pq_7[1]; - uint32_t _prmt_381; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_381) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); - pq_7[0] = _prmt_381; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_399; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_399) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_399; +#else + uint32_t _prmt_385; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_385) : "r"(pw_0[3]), "r"(pv_7), "r"(sel_0)); + pq_7[0] = _prmt_385; +#endif int t_q_7 = 32 + 8 * th + 6 + odd_0; if (t_q_7 < clen) { { - auto _gst_382 = draw_base + (t_q_7 * rs_g - odd_0); - __builtin_assume(__isGlobal(_gst_382)); - *(reinterpret_cast(_gst_382) + (0)) = pq_7[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_400 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_400)); + *(reinterpret_cast(_gst_400) + (0)) = pq_7[0]; +#else + auto _gst_386 = draw_base + (t_q_7 * rs_g - odd_0); + __builtin_assume(__isGlobal(_gst_386)); + *(reinterpret_cast(_gst_386) + (0)) = pq_7[0]; +#endif } } int t_a_8 = 32 + 8 * th; @@ -4404,47 +4877,99 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_0) : "f"(pa01_0), "f"(pa23_0)); float nm_t_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_1) : "f"(s_a_0), "f"(pa_0)); - float _sub_383; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_383) : "f"(s_a_0), "f"(nm_t_1)); +#if __CUDA_ARCH__ == 1030 + float _sub_401; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_401) : "f"(s_a_0), "f"(nm_t_1)); +#else + float _sub_387; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_387) : "f"(s_a_0), "f"(nm_t_1)); +#endif float nm_es_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_383), "f"(pa_0)); - float _sub_384; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_384) : "f"(pa_0), "f"(nm_t_1)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_401), "f"(pa_0)); + float _sub_402; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_402) : "f"(pa_0), "f"(nm_t_1)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_1) : "f"(_sub_387), "f"(pa_0)); + float _sub_388; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_388) : "f"(pa_0), "f"(nm_t_1)); +#endif float nm_ex_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_384), "f"(s_a_0)); - float _abs_386; - asm("abs.f32 %0, %1;" : "=f"(_abs_386) : "f"(s_a_0)); - float _abs_387; - asm("abs.f32 %0, %1;" : "=f"(_abs_387) : "f"(pa_0)); - uint32_t _setp_385_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_385_bits) : "f"(_abs_386), "f"(_abs_387)); - bool _setp_385 = _setp_385_bits != 0; - float nm_err_1 = ((_setp_385) ? nm_es_1 : nm_ex_1); - float _add_388; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_388) : "f"(c_a_0), "f"(nm_err_1)); - c_a_0 = _add_388; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_402), "f"(s_a_0)); + float _abs_404; + asm("abs.f32 %0, %1;" : "=f"(_abs_404) : "f"(s_a_0)); + float _abs_405; + asm("abs.f32 %0, %1;" : "=f"(_abs_405) : "f"(pa_0)); + uint32_t _setp_403_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_403_bits) : "f"(_abs_404), "f"(_abs_405)); + bool _setp_403 = _setp_403_bits != 0; + float nm_err_1 = ((_setp_403) ? nm_es_1 : nm_ex_1); + float _add_406; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_406) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_406; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_1) : "f"(_sub_388), "f"(s_a_0)); + float _abs_390; + asm("abs.f32 %0, %1;" : "=f"(_abs_390) : "f"(s_a_0)); + float _abs_391; + asm("abs.f32 %0, %1;" : "=f"(_abs_391) : "f"(pa_0)); + uint32_t _setp_389_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_389_bits) : "f"(_abs_390), "f"(_abs_391)); + bool _setp_389 = _setp_389_bits != 0; + float nm_err_1 = ((_setp_389) ? nm_es_1 : nm_ex_1); + float _add_392; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_392) : "f"(c_a_0), "f"(nm_err_1)); + c_a_0 = _add_392; +#endif s_a_0 = nm_t_1; float nm_t_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_2) : "f"(s_b_0), "f"(pb_0)); - float _sub_389; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_389) : "f"(s_b_0), "f"(nm_t_2)); +#if __CUDA_ARCH__ == 1030 + float _sub_407; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_407) : "f"(s_b_0), "f"(nm_t_2)); +#else + float _sub_393; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_393) : "f"(s_b_0), "f"(nm_t_2)); +#endif float nm_es_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_389), "f"(pb_0)); - float _sub_390; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_390) : "f"(pb_0), "f"(nm_t_2)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_407), "f"(pb_0)); + float _sub_408; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_408) : "f"(pb_0), "f"(nm_t_2)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_2) : "f"(_sub_393), "f"(pb_0)); + float _sub_394; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_394) : "f"(pb_0), "f"(nm_t_2)); +#endif float nm_ex_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_390), "f"(s_b_0)); - float _abs_392; - asm("abs.f32 %0, %1;" : "=f"(_abs_392) : "f"(s_b_0)); - float _abs_393; - asm("abs.f32 %0, %1;" : "=f"(_abs_393) : "f"(pb_0)); - uint32_t _setp_391_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_391_bits) : "f"(_abs_392), "f"(_abs_393)); - bool _setp_391 = _setp_391_bits != 0; - float nm_err_2 = ((_setp_391) ? nm_es_2 : nm_ex_2); - float _add_394; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_394) : "f"(c_b_0), "f"(nm_err_2)); - c_b_0 = _add_394; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_408), "f"(s_b_0)); + float _abs_410; + asm("abs.f32 %0, %1;" : "=f"(_abs_410) : "f"(s_b_0)); + float _abs_411; + asm("abs.f32 %0, %1;" : "=f"(_abs_411) : "f"(pb_0)); + uint32_t _setp_409_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_409_bits) : "f"(_abs_410), "f"(_abs_411)); + bool _setp_409 = _setp_409_bits != 0; + float nm_err_2 = ((_setp_409) ? nm_es_2 : nm_ex_2); + float _add_412; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_412) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_412; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_2) : "f"(_sub_394), "f"(s_b_0)); + float _abs_396; + asm("abs.f32 %0, %1;" : "=f"(_abs_396) : "f"(s_b_0)); + float _abs_397; + asm("abs.f32 %0, %1;" : "=f"(_abs_397) : "f"(pb_0)); + uint32_t _setp_395_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_395_bits) : "f"(_abs_396), "f"(_abs_397)); + bool _setp_395 = _setp_395_bits != 0; + float nm_err_2 = ((_setp_395) ? nm_es_2 : nm_ex_2); + float _add_398; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_398) : "f"(c_b_0), "f"(nm_err_2)); + c_b_0 = _add_398; +#endif s_b_0 = nm_t_2; gacc[0] = s_a_0; gacc[1] = c_a_0; @@ -4452,8 +4977,13 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d gacc[3] = c_b_0; off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_395 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_395), "f"(off_acc[0]) : "memory"); +#if __CUDA_ARCH__ == 1030 + uint32_t _addr_413 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_413), "f"(off_acc[0]) : "memory"); +#else + uint32_t _addr_399 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_399), "f"(off_acc[0]) : "memory"); +#endif } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -4479,9 +5009,15 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm volatile("tcgen05.fence::after_thread_sync;"); mbarrier_wait(epi_full_addr + (2 + th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_297[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_1 = r_297[0]; +#if __CUDA_ARCH__ == 1030 + float r_276[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1 = r_276[0]; +#else + float r_255[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_255[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 2048 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_1 = r_255[0]; +#endif float gc_2 = gn_1 - 21.640425f; int eb_1 = 81920 + th * -32768; int ew_1 = eb_1 >> 2; @@ -4490,368 +5026,831 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int gb8_1 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_1 * 128); int kb8_1 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_1 * 128); float dbp_1[8]; - float qo_1[8]; - tmem_ld_x8(&qo_1[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); - float ko_1[8]; - tmem_ld_x8(&ko_1[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); - float qdg_1[8]; - tmem_ld_x8(&qdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); - float kdg_1[8]; - tmem_ld_x8(&kdg_1[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); +#if __CUDA_ARCH__ == 1030 + float qo_0[8]; + tmem_ld_x8(&qo_0[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); + float ko_0[8]; + tmem_ld_x8(&ko_0[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); + float qdg_0[8]; + tmem_ld_x8(&qdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); + float kdg_0[8]; + tmem_ld_x8(&kdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); +#else + float qo_2[8]; + tmem_ld_x8(&qo_2[0], taddr + (unsigned int)row_base + (unsigned int)(64 + 8 * th)); + float ko_2[8]; + tmem_ld_x8(&ko_2[0], taddr + (unsigned int)row_base + (unsigned int)(80 + 8 * th)); + float qdg_2[8]; + tmem_ld_x8(&qdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)); + float kdg_2[8]; + tmem_ld_x8(&kdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(112 + 8 * th)); +#endif float dkt_2[8]; tmem_ld_x8(&dkt_2[0], taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th)); - float btv_1[8]; +#if __CUDA_ARCH__ == 1030 + float btv_0[8]; +#else + float btv_2[8]; +#endif asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1[0])), "=r"(*reinterpret_cast(&btv_1[(0) + 1])), "=r"(*reinterpret_cast(&btv_1[(0) + 2])), "=r"(*reinterpret_cast(&btv_1[(0) + 3])) +#if __CUDA_ARCH__ == 1030 + : "=r"(*reinterpret_cast(&btv_0[0])), "=r"(*reinterpret_cast(&btv_0[(0) + 1])), "=r"(*reinterpret_cast(&btv_0[(0) + 2])), "=r"(*reinterpret_cast(&btv_0[(0) + 3])) +#else + : "=r"(*reinterpret_cast(&btv_2[0])), "=r"(*reinterpret_cast(&btv_2[(0) + 1])), "=r"(*reinterpret_cast(&btv_2[(0) + 2])), "=r"(*reinterpret_cast(&btv_2[(0) + 3])) +#endif : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_1[4])), "=r"(*reinterpret_cast(&btv_1[(4) + 1])), "=r"(*reinterpret_cast(&btv_1[(4) + 2])), "=r"(*reinterpret_cast(&btv_1[(4) + 3])) +#if __CUDA_ARCH__ == 1030 + : "=r"(*reinterpret_cast(&btv_0[4])), "=r"(*reinterpret_cast(&btv_0[(4) + 1])), "=r"(*reinterpret_cast(&btv_0[(4) + 2])), "=r"(*reinterpret_cast(&btv_0[(4) + 3])) +#else + : "=r"(*reinterpret_cast(&btv_2[4])), "=r"(*reinterpret_cast(&btv_2[(4) + 1])), "=r"(*reinterpret_cast(&btv_2[(4) + 2])), "=r"(*reinterpret_cast(&btv_2[(4) + 3])) +#endif : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (16 + 8 * th) + 16))); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_1[1]; s_run_1[0] = 0.0f; - int t_16 = 16 + 8 * th + 7; - float r_298[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_16 = r_298[0]; - unsigned int r_299[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_16 = __uint_as_float(r_299[0] << 16); - unsigned int r_300[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_16 = __uint_as_float(r_300[0] << 16); - float q_t_1_16 = ((t_16 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1 + 896 + d_lane]; - float dkf_16 = pool_f32[ew_1 + 1920 + d_lane]; - float dgf_16 = pool_f32[ew_1 + 2944 + d_lane]; - float bt_t_16 = btv_1[7]; - float e_off_16 = approx_exp2(g_t_16 - gn_1); - float e_dg_16 = approx_exp2(g_t_16 - gc_2); - float dq2_16 = __fmaf_rn(qo_1[7], e_off_16, qdg_1[7] * e_dg_16); - float dk2_16 = __fmaf_rn(ko_1[7], e_off_16, kdg_1[7] * e_dg_16); - dbp_1[7] = dk2_16 * k_t_16; - float dk2b_16 = dk2_16 * bt_t_16; - float dq_o_16 = dq2_16 + dqf_16; - float dk_o_16 = dk2b_16 + dkf_16 + dkt_2[7]; - float dg_o_16 = __fmaf_rn(q_t_1_16, dq2_16, (dk2b_16 - dkt_2[7]) * k_t_16) + dgf_16; - s_run_1[0] = s_run_1[0] + dg_o_16; +#if __CUDA_ARCH__ == 1030 + int t_8 = 16 + 8 * th + 7; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_8 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_8 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_8 = __uint_as_float(r_279[0] << 16); + float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); + float bt_t_8 = btv_0[7]; + float e_off_8 = approx_exp2(g_t_8 - gn_1); + float e_dg_8 = approx_exp2(g_t_8 - gc_2); + float dq2_8 = __fmaf_rn(qo_0[7], e_off_8, qdg_0[7] * e_dg_8); + float dk2_8 = __fmaf_rn(ko_0[7], e_off_8, kdg_0[7] * e_dg_8); + dbp_1[7] = dk2_8 * k_t_8; + float dk2b_8 = dk2_8 * bt_t_8; + float qe_7 = q_t_1_8 * e_dg_8; + float kp_19 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_8) * kdg_0[7]); + float dqf_8 = pool_f32[ew_1 + 896 + d_lane]; + float dkf_8 = pool_f32[ew_1 + 1920 + d_lane]; + float dgf_8 = pool_f32[ew_1 + 2944 + d_lane]; + float dq_o_8 = dq2_8 + dqf_8; + float dk_o_8 = dk2b_8 + dkf_8 + dkt_2[7]; + float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_2[7], k_t_8, cr_7)) + dgf_8; + s_run_1[0] = s_run_1[0] + dg_o_8; +#else + int t_0 = 16 + 8 * th + 7; + float r_256[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_0 = r_256[0]; + unsigned int r_257[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_257[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_0 = __uint_as_float(r_257[0] << 16); + unsigned int r_258[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_258[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_0 = __uint_as_float(r_258[0] << 16); + float q_t_1_0 = ((t_0 >= clen) ? 0.0f : q_t_0); + float bt_t_0 = btv_2[7]; + float e_off_0 = approx_exp2(g_t_0 - gn_1); + float e_dg_0 = approx_exp2(g_t_0 - gc_2); + float dq2_0 = __fmaf_rn(qo_2[7], e_off_0, qdg_2[7] * e_dg_0); + float dk2_0 = __fmaf_rn(ko_2[7], e_off_0, kdg_2[7] * e_dg_0); + dbp_1[7] = dk2_0 * k_t_0; + float dk2b_0 = dk2_0 * bt_t_0; + float qe_2 = q_t_1_0 * e_dg_0; + float kp_13 = k_t_0 * bt_t_0; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_13; + float kbe_2 = (float)kb_2 * e_dg_0; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg_2[7], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_13 * e_dg_0) * kdg_2[7]); + float dqf_0 = pool_f32[ew_1 + 896 + d_lane]; + float dkf_0 = pool_f32[ew_1 + 1920 + d_lane]; + float dgf_0 = pool_f32[ew_1 + 2944 + d_lane]; + float dq_o_0 = dq2_0 + dqf_0; + float dk_o_0 = dk2b_0 + dkf_0 + dkt_2[7]; + float dg_o_0 = __fmaf_rn(q_t_1_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_2[7], k_t_0, cr_2)) + dgf_0; + s_run_1[0] = s_run_1[0] + dg_o_0; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 7)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_16 = t_16 * qrs; - if (t_16 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_8 = t_8 * qrs; + if (t_8 < clen) { +#else + int qoff_0 = t_0 * qrs; + if (t_0 < clen) { +#endif { - auto _gst_396 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_396)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_396) + (0)) = (__nv_bfloat16)dq_o_16; +#if __CUDA_ARCH__ == 1030 + auto _gst_414 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_414)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_414) + (0)) = (__nv_bfloat16)dq_o_8; +#else + auto _gst_400 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_400)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_0; +#endif } { - auto _gst_397 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_397)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_397) + (0)) = (__nv_bfloat16)dk_o_16; +#if __CUDA_ARCH__ == 1030 + auto _gst_415 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_415)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_415) + (0)) = (__nv_bfloat16)dk_o_8; +#else + auto _gst_401 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_401)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_0; +#endif } } - int t_17 = 16 + 8 * th + 6; - float r_301[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_17 = r_301[0]; - unsigned int r_302[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_17 = __uint_as_float(r_302[0] << 16); - unsigned int r_303[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_17 = __uint_as_float(r_303[0] << 16); - float q_t_1_17 = ((t_17 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1 + 768 + d_lane]; - float dkf_17 = pool_f32[ew_1 + 1792 + d_lane]; - float dgf_17 = pool_f32[ew_1 + 2816 + d_lane]; - float bt_t_17 = btv_1[6]; - float e_off_17 = approx_exp2(g_t_17 - gn_1); - float e_dg_17 = approx_exp2(g_t_17 - gc_2); - float dq2_17 = __fmaf_rn(qo_1[6], e_off_17, qdg_1[6] * e_dg_17); - float dk2_17 = __fmaf_rn(ko_1[6], e_off_17, kdg_1[6] * e_dg_17); - dbp_1[6] = dk2_17 * k_t_17; - float dk2b_17 = dk2_17 * bt_t_17; - float dq_o_17 = dq2_17 + dqf_17; - float dk_o_17 = dk2b_17 + dkf_17 + dkt_2[6]; - float dg_o_17 = __fmaf_rn(q_t_1_17, dq2_17, (dk2b_17 - dkt_2[6]) * k_t_17) + dgf_17; - s_run_1[0] = s_run_1[0] + dg_o_17; +#if __CUDA_ARCH__ == 1030 + int t_9 = 16 + 8 * th + 6; + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_9 = r_280[0]; + unsigned int r_281[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_9 = __uint_as_float(r_281[0] << 16); + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_9 = __uint_as_float(r_282[0] << 16); + float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); + float bt_t_9 = btv_0[6]; + float e_off_9 = approx_exp2(g_t_9 - gn_1); + float e_dg_9 = approx_exp2(g_t_9 - gc_2); + float dq2_9 = __fmaf_rn(qo_0[6], e_off_9, qdg_0[6] * e_dg_9); + float dk2_9 = __fmaf_rn(ko_0[6], e_off_9, kdg_0[6] * e_dg_9); + dbp_1[6] = dk2_9 * k_t_9; + float dk2b_9 = dk2_9 * bt_t_9; + float qe_8 = q_t_1_9 * e_dg_9; + float kp_20 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_9) * kdg_0[6]); + float dqf_9 = pool_f32[ew_1 + 768 + d_lane]; + float dkf_9 = pool_f32[ew_1 + 1792 + d_lane]; + float dgf_9 = pool_f32[ew_1 + 2816 + d_lane]; + float dq_o_9 = dq2_9 + dqf_9; + float dk_o_9 = dk2b_9 + dkf_9 + dkt_2[6]; + float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_2[6], k_t_9, cr_8)) + dgf_9; + s_run_1[0] = s_run_1[0] + dg_o_9; +#else + int t_1_1 = 16 + 8 * th + 6; + float r_259[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_1_1 = r_259[0]; + unsigned int r_260[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_260[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_1_1 = __uint_as_float(r_260[0] << 16); + unsigned int r_261[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_261[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_1_2 = __uint_as_float(r_261[0] << 16); + float q_t_1_1_1 = ((t_1_1 >= clen) ? 0.0f : q_t_1_2); + float bt_t_1_1 = btv_2[6]; + float e_off_1_1 = approx_exp2(g_t_1_1 - gn_1); + float e_dg_1_1 = approx_exp2(g_t_1_1 - gc_2); + float dq2_1_1 = __fmaf_rn(qo_2[6], e_off_1_1, qdg_2[6] * e_dg_1_1); + float dk2_1_1 = __fmaf_rn(ko_2[6], e_off_1_1, kdg_2[6] * e_dg_1_1); + dbp_1[6] = dk2_1_1 * k_t_1_1; + float dk2b_1_1 = dk2_1_1 * bt_t_1_1; + float qe_0 = q_t_1_1_1 * e_dg_1_1; + float kp_12_1 = k_t_1_1 * bt_t_1_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_12_1; + float kbe_0 = (float)kb_0 * e_dg_1_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg_2[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_12_1 * e_dg_1_1) * kdg_2[6]); + float dqf_1_1 = pool_f32[ew_1 + 768 + d_lane]; + float dkf_1_1 = pool_f32[ew_1 + 1792 + d_lane]; + float dgf_1_1 = pool_f32[ew_1 + 2816 + d_lane]; + float dq_o_1_1 = dq2_1_1 + dqf_1_1; + float dk_o_1_1 = dk2b_1_1 + dkf_1_1 + dkt_2[6]; + float dg_o_1_1 = __fmaf_rn(q_t_1_1_1, dq2_1_1, __fmaf_rn(dk2b_1_1 - dkt_2[6], k_t_1_1, cr_0)) + dgf_1_1; + s_run_1[0] = s_run_1[0] + dg_o_1_1; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 6)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_17 = t_17 * qrs; - if (t_17 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_9 = t_9 * qrs; + if (t_9 < clen) { +#else + int qoff_1_1 = t_1_1 * qrs; + if (t_1_1 < clen) { +#endif { - auto _gst_398 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_398)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_398) + (0)) = (__nv_bfloat16)dq_o_17; +#if __CUDA_ARCH__ == 1030 + auto _gst_416 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_416)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_416) + (0)) = (__nv_bfloat16)dq_o_9; +#else + auto _gst_402 = dq_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_402)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_1_1; +#endif } { - auto _gst_399 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_399)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_399) + (0)) = (__nv_bfloat16)dk_o_17; +#if __CUDA_ARCH__ == 1030 + auto _gst_417 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_417)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_417) + (0)) = (__nv_bfloat16)dk_o_9; +#else + auto _gst_403 = dk_base + qoff_1_1; + __builtin_assume(__isGlobal(_gst_403)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_1_1; +#endif } } - int t_18 = 16 + 8 * th + 5; - float r_304[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_18 = r_304[0]; - unsigned int r_305[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_18 = __uint_as_float(r_305[0] << 16); - unsigned int r_306[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_18 = __uint_as_float(r_306[0] << 16); - float q_t_1_18 = ((t_18 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1 + 640 + d_lane]; - float dkf_18 = pool_f32[ew_1 + 1664 + d_lane]; - float dgf_18 = pool_f32[ew_1 + 2688 + d_lane]; - float bt_t_18 = btv_1[5]; - float e_off_18 = approx_exp2(g_t_18 - gn_1); - float e_dg_18 = approx_exp2(g_t_18 - gc_2); - float dq2_18 = __fmaf_rn(qo_1[5], e_off_18, qdg_1[5] * e_dg_18); - float dk2_18 = __fmaf_rn(ko_1[5], e_off_18, kdg_1[5] * e_dg_18); - dbp_1[5] = dk2_18 * k_t_18; - float dk2b_18 = dk2_18 * bt_t_18; - float dq_o_18 = dq2_18 + dqf_18; - float dk_o_18 = dk2b_18 + dkf_18 + dkt_2[5]; - float dg_o_18 = __fmaf_rn(q_t_1_18, dq2_18, (dk2b_18 - dkt_2[5]) * k_t_18) + dgf_18; - s_run_1[0] = s_run_1[0] + dg_o_18; +#if __CUDA_ARCH__ == 1030 + int t_10 = 16 + 8 * th + 5; + float r_283[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_10 = r_283[0]; + unsigned int r_284[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_10 = __uint_as_float(r_284[0] << 16); + unsigned int r_285[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_10 = __uint_as_float(r_285[0] << 16); + float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); + float bt_t_10 = btv_0[5]; + float e_off_10 = approx_exp2(g_t_10 - gn_1); + float e_dg_10 = approx_exp2(g_t_10 - gc_2); + float dq2_10 = __fmaf_rn(qo_0[5], e_off_10, qdg_0[5] * e_dg_10); + float dk2_10 = __fmaf_rn(ko_0[5], e_off_10, kdg_0[5] * e_dg_10); + dbp_1[5] = dk2_10 * k_t_10; + float dk2b_10 = dk2_10 * bt_t_10; + float qe_9 = q_t_1_10 * e_dg_10; + float kp_21 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_10) * kdg_0[5]); + float dqf_10 = pool_f32[ew_1 + 640 + d_lane]; + float dkf_10 = pool_f32[ew_1 + 1664 + d_lane]; + float dgf_10 = pool_f32[ew_1 + 2688 + d_lane]; + float dq_o_10 = dq2_10 + dqf_10; + float dk_o_10 = dk2b_10 + dkf_10 + dkt_2[5]; + float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_2[5], k_t_10, cr_9)) + dgf_10; + s_run_1[0] = s_run_1[0] + dg_o_10; +#else + int t_2 = 16 + 8 * th + 5; + float r_262[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_2 = r_262[0]; + unsigned int r_263[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_263[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_2 = __uint_as_float(r_263[0] << 16); + unsigned int r_264[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_264[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_2_1 = __uint_as_float(r_264[0] << 16); + float q_t_1_2_1 = ((t_2 >= clen) ? 0.0f : q_t_2_1); + float bt_t_2 = btv_2[5]; + float e_off_2 = approx_exp2(g_t_2 - gn_1); + float e_dg_2 = approx_exp2(g_t_2 - gc_2); + float dq2_2 = __fmaf_rn(qo_2[5], e_off_2, qdg_2[5] * e_dg_2); + float dk2_2 = __fmaf_rn(ko_2[5], e_off_2, kdg_2[5] * e_dg_2); + dbp_1[5] = dk2_2 * k_t_2; + float dk2b_2 = dk2_2 * bt_t_2; + float qe_1_1 = q_t_1_2_1 * e_dg_2; + float kp_13_1 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1_1 = (__nv_bfloat16)kp_13_1; + float kbe_1_1 = (float)kb_1_1 * e_dg_2; + float cr_1_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1_1) & 4294959104u) - qe_1_1, qdg_2[5], (__uint_as_float(__as_u32(kbe_1_1) & 4294959104u) - kp_13_1 * e_dg_2) * kdg_2[5]); + float dqf_2 = pool_f32[ew_1 + 640 + d_lane]; + float dkf_2 = pool_f32[ew_1 + 1664 + d_lane]; + float dgf_2 = pool_f32[ew_1 + 2688 + d_lane]; + float dq_o_2 = dq2_2 + dqf_2; + float dk_o_2 = dk2b_2 + dkf_2 + dkt_2[5]; + float dg_o_2 = __fmaf_rn(q_t_1_2_1, dq2_2, __fmaf_rn(dk2b_2 - dkt_2[5], k_t_2, cr_1_1)) + dgf_2; + s_run_1[0] = s_run_1[0] + dg_o_2; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 5)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_18 = t_18 * qrs; - if (t_18 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_10 = t_10 * qrs; + if (t_10 < clen) { +#else + int qoff_2 = t_2 * qrs; + if (t_2 < clen) { +#endif { - auto _gst_400 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_400)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_400) + (0)) = (__nv_bfloat16)dq_o_18; +#if __CUDA_ARCH__ == 1030 + auto _gst_418 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_418)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_418) + (0)) = (__nv_bfloat16)dq_o_10; +#else + auto _gst_404 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_404)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_2; +#endif } { - auto _gst_401 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_401)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_401) + (0)) = (__nv_bfloat16)dk_o_18; +#if __CUDA_ARCH__ == 1030 + auto _gst_419 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_419)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_419) + (0)) = (__nv_bfloat16)dk_o_10; +#else + auto _gst_405 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_405)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_2; +#endif } } - int t_19 = 16 + 8 * th + 4; - float r_307[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_19 = r_307[0]; - unsigned int r_308[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_19 = __uint_as_float(r_308[0] << 16); - unsigned int r_309[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_19 = __uint_as_float(r_309[0] << 16); - float q_t_1_19 = ((t_19 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1 + 512 + d_lane]; - float dkf_19 = pool_f32[ew_1 + 1536 + d_lane]; - float dgf_19 = pool_f32[ew_1 + 2560 + d_lane]; - float bt_t_19 = btv_1[4]; - float e_off_19 = approx_exp2(g_t_19 - gn_1); - float e_dg_19 = approx_exp2(g_t_19 - gc_2); - float dq2_19 = __fmaf_rn(qo_1[4], e_off_19, qdg_1[4] * e_dg_19); - float dk2_19 = __fmaf_rn(ko_1[4], e_off_19, kdg_1[4] * e_dg_19); - dbp_1[4] = dk2_19 * k_t_19; - float dk2b_19 = dk2_19 * bt_t_19; - float dq_o_19 = dq2_19 + dqf_19; - float dk_o_19 = dk2b_19 + dkf_19 + dkt_2[4]; - float dg_o_19 = __fmaf_rn(q_t_1_19, dq2_19, (dk2b_19 - dkt_2[4]) * k_t_19) + dgf_19; - s_run_1[0] = s_run_1[0] + dg_o_19; +#if __CUDA_ARCH__ == 1030 + int t_11 = 16 + 8 * th + 4; + float r_286[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_11 = r_286[0]; + unsigned int r_287[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_11 = __uint_as_float(r_287[0] << 16); + unsigned int r_288[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_11 = __uint_as_float(r_288[0] << 16); + float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); + float bt_t_11 = btv_0[4]; + float e_off_11 = approx_exp2(g_t_11 - gn_1); + float e_dg_11 = approx_exp2(g_t_11 - gc_2); + float dq2_11 = __fmaf_rn(qo_0[4], e_off_11, qdg_0[4] * e_dg_11); + float dk2_11 = __fmaf_rn(ko_0[4], e_off_11, kdg_0[4] * e_dg_11); + dbp_1[4] = dk2_11 * k_t_11; + float dk2b_11 = dk2_11 * bt_t_11; + float qe_10 = q_t_1_11 * e_dg_11; + float kp_22 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_11) * kdg_0[4]); + float dqf_11 = pool_f32[ew_1 + 512 + d_lane]; + float dkf_11 = pool_f32[ew_1 + 1536 + d_lane]; + float dgf_11 = pool_f32[ew_1 + 2560 + d_lane]; + float dq_o_11 = dq2_11 + dqf_11; + float dk_o_11 = dk2b_11 + dkf_11 + dkt_2[4]; + float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_2[4], k_t_11, cr_10)) + dgf_11; + s_run_1[0] = s_run_1[0] + dg_o_11; +#else + int t_3 = 16 + 8 * th + 4; + float r_265[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_3 = r_265[0]; + unsigned int r_266[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_266[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_3 = __uint_as_float(r_266[0] << 16); + unsigned int r_267[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_267[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_3 = __uint_as_float(r_267[0] << 16); + float q_t_1_3 = ((t_3 >= clen) ? 0.0f : q_t_3); + float bt_t_3 = btv_2[4]; + float e_off_3 = approx_exp2(g_t_3 - gn_1); + float e_dg_3 = approx_exp2(g_t_3 - gc_2); + float dq2_3 = __fmaf_rn(qo_2[4], e_off_3, qdg_2[4] * e_dg_3); + float dk2_3 = __fmaf_rn(ko_2[4], e_off_3, kdg_2[4] * e_dg_3); + dbp_1[4] = dk2_3 * k_t_3; + float dk2b_3 = dk2_3 * bt_t_3; + float qe_2_1 = q_t_1_3 * e_dg_3; + float kp_14 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2_1 = (__nv_bfloat16)kp_14; + float kbe_2_1 = (float)kb_2_1 * e_dg_3; + float cr_2_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_2_1) & 4294959104u) - qe_2_1, qdg_2[4], (__uint_as_float(__as_u32(kbe_2_1) & 4294959104u) - kp_14 * e_dg_3) * kdg_2[4]); + float dqf_3 = pool_f32[ew_1 + 512 + d_lane]; + float dkf_3 = pool_f32[ew_1 + 1536 + d_lane]; + float dgf_3 = pool_f32[ew_1 + 2560 + d_lane]; + float dq_o_3 = dq2_3 + dqf_3; + float dk_o_3 = dk2b_3 + dkf_3 + dkt_2[4]; + float dg_o_3 = __fmaf_rn(q_t_1_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_2[4], k_t_3, cr_2_1)) + dgf_3; + s_run_1[0] = s_run_1[0] + dg_o_3; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 4)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_19 = t_19 * qrs; - if (t_19 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_11 = t_11 * qrs; + if (t_11 < clen) { +#else + int qoff_3 = t_3 * qrs; + if (t_3 < clen) { +#endif { - auto _gst_402 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_402)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_402) + (0)) = (__nv_bfloat16)dq_o_19; +#if __CUDA_ARCH__ == 1030 + auto _gst_420 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_420)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_420) + (0)) = (__nv_bfloat16)dq_o_11; +#else + auto _gst_406 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_406)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_3; +#endif } { - auto _gst_403 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_403)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_403) + (0)) = (__nv_bfloat16)dk_o_19; +#if __CUDA_ARCH__ == 1030 + auto _gst_421 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_421)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_421) + (0)) = (__nv_bfloat16)dk_o_11; +#else + auto _gst_407 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_407)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_3; +#endif } } - int t_20 = 16 + 8 * th + 3; - float r_310[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_20 = r_310[0]; - unsigned int r_311[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_20 = __uint_as_float(r_311[0] << 16); - unsigned int r_312[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_20 = __uint_as_float(r_312[0] << 16); - float q_t_1_20 = ((t_20 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1 + 384 + d_lane]; - float dkf_20 = pool_f32[ew_1 + 1408 + d_lane]; - float dgf_20 = pool_f32[ew_1 + 2432 + d_lane]; - float bt_t_20 = btv_1[3]; - float e_off_20 = approx_exp2(g_t_20 - gn_1); - float e_dg_20 = approx_exp2(g_t_20 - gc_2); - float dq2_20 = __fmaf_rn(qo_1[3], e_off_20, qdg_1[3] * e_dg_20); - float dk2_20 = __fmaf_rn(ko_1[3], e_off_20, kdg_1[3] * e_dg_20); - dbp_1[3] = dk2_20 * k_t_20; - float dk2b_20 = dk2_20 * bt_t_20; - float dq_o_20 = dq2_20 + dqf_20; - float dk_o_20 = dk2b_20 + dkf_20 + dkt_2[3]; - float dg_o_20 = __fmaf_rn(q_t_1_20, dq2_20, (dk2b_20 - dkt_2[3]) * k_t_20) + dgf_20; - s_run_1[0] = s_run_1[0] + dg_o_20; +#if __CUDA_ARCH__ == 1030 + int t_12 = 16 + 8 * th + 3; + float r_289[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_12 = r_289[0]; + unsigned int r_290[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_12 = __uint_as_float(r_290[0] << 16); + unsigned int r_291[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_12 = __uint_as_float(r_291[0] << 16); + float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); + float bt_t_12 = btv_0[3]; + float e_off_12 = approx_exp2(g_t_12 - gn_1); + float e_dg_12 = approx_exp2(g_t_12 - gc_2); + float dq2_12 = __fmaf_rn(qo_0[3], e_off_12, qdg_0[3] * e_dg_12); + float dk2_12 = __fmaf_rn(ko_0[3], e_off_12, kdg_0[3] * e_dg_12); + dbp_1[3] = dk2_12 * k_t_12; + float dk2b_12 = dk2_12 * bt_t_12; + float qe_11 = q_t_1_12 * e_dg_12; + float kp_23 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_12) * kdg_0[3]); + float dqf_12 = pool_f32[ew_1 + 384 + d_lane]; + float dkf_12 = pool_f32[ew_1 + 1408 + d_lane]; + float dgf_12 = pool_f32[ew_1 + 2432 + d_lane]; + float dq_o_12 = dq2_12 + dqf_12; + float dk_o_12 = dk2b_12 + dkf_12 + dkt_2[3]; + float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_2[3], k_t_12, cr_11)) + dgf_12; + s_run_1[0] = s_run_1[0] + dg_o_12; +#else + int t_4 = 16 + 8 * th + 3; + float r_268[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_4 = r_268[0]; + unsigned int r_269[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_269[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_4 = __uint_as_float(r_269[0] << 16); + unsigned int r_270[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_270[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_4 = __uint_as_float(r_270[0] << 16); + float q_t_1_4 = ((t_4 >= clen) ? 0.0f : q_t_4); + float bt_t_4 = btv_2[3]; + float e_off_4 = approx_exp2(g_t_4 - gn_1); + float e_dg_4 = approx_exp2(g_t_4 - gc_2); + float dq2_4 = __fmaf_rn(qo_2[3], e_off_4, qdg_2[3] * e_dg_4); + float dk2_4 = __fmaf_rn(ko_2[3], e_off_4, kdg_2[3] * e_dg_4); + dbp_1[3] = dk2_4 * k_t_4; + float dk2b_4 = dk2_4 * bt_t_4; + float qe_3 = q_t_1_4 * e_dg_4; + float kp_15 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_15; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg_2[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_15 * e_dg_4) * kdg_2[3]); + float dqf_4 = pool_f32[ew_1 + 384 + d_lane]; + float dkf_4 = pool_f32[ew_1 + 1408 + d_lane]; + float dgf_4 = pool_f32[ew_1 + 2432 + d_lane]; + float dq_o_4 = dq2_4 + dqf_4; + float dk_o_4 = dk2b_4 + dkf_4 + dkt_2[3]; + float dg_o_4 = __fmaf_rn(q_t_1_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_2[3], k_t_4, cr_3)) + dgf_4; + s_run_1[0] = s_run_1[0] + dg_o_4; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 3)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_20 = t_20 * qrs; - if (t_20 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_12 = t_12 * qrs; + if (t_12 < clen) { +#else + int qoff_4 = t_4 * qrs; + if (t_4 < clen) { +#endif { - auto _gst_404 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_404)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_404) + (0)) = (__nv_bfloat16)dq_o_20; +#if __CUDA_ARCH__ == 1030 + auto _gst_422 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_422)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_422) + (0)) = (__nv_bfloat16)dq_o_12; +#else + auto _gst_408 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_408)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_4; +#endif } { - auto _gst_405 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_405)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_405) + (0)) = (__nv_bfloat16)dk_o_20; +#if __CUDA_ARCH__ == 1030 + auto _gst_423 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_423)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_423) + (0)) = (__nv_bfloat16)dk_o_12; +#else + auto _gst_409 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_409)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_4; +#endif } } - int t_21 = 16 + 8 * th + 2; - float r_313[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_21 = r_313[0]; - unsigned int r_314[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_21 = __uint_as_float(r_314[0] << 16); - unsigned int r_315[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_21 = __uint_as_float(r_315[0] << 16); - float q_t_1_21 = ((t_21 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1 + 256 + d_lane]; - float dkf_21 = pool_f32[ew_1 + 1280 + d_lane]; - float dgf_21 = pool_f32[ew_1 + 2304 + d_lane]; - float bt_t_21 = btv_1[2]; - float e_off_21 = approx_exp2(g_t_21 - gn_1); - float e_dg_21 = approx_exp2(g_t_21 - gc_2); - float dq2_21 = __fmaf_rn(qo_1[2], e_off_21, qdg_1[2] * e_dg_21); - float dk2_21 = __fmaf_rn(ko_1[2], e_off_21, kdg_1[2] * e_dg_21); - dbp_1[2] = dk2_21 * k_t_21; - float dk2b_21 = dk2_21 * bt_t_21; - float dq_o_21 = dq2_21 + dqf_21; - float dk_o_21 = dk2b_21 + dkf_21 + dkt_2[2]; - float dg_o_21 = __fmaf_rn(q_t_1_21, dq2_21, (dk2b_21 - dkt_2[2]) * k_t_21) + dgf_21; - s_run_1[0] = s_run_1[0] + dg_o_21; +#if __CUDA_ARCH__ == 1030 + int t_13 = 16 + 8 * th + 2; + float r_292[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_13 = r_292[0]; + unsigned int r_293[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_13 = __uint_as_float(r_293[0] << 16); + unsigned int r_294[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_13 = __uint_as_float(r_294[0] << 16); + float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); + float bt_t_13 = btv_0[2]; + float e_off_13 = approx_exp2(g_t_13 - gn_1); + float e_dg_13 = approx_exp2(g_t_13 - gc_2); + float dq2_13 = __fmaf_rn(qo_0[2], e_off_13, qdg_0[2] * e_dg_13); + float dk2_13 = __fmaf_rn(ko_0[2], e_off_13, kdg_0[2] * e_dg_13); + dbp_1[2] = dk2_13 * k_t_13; + float dk2b_13 = dk2_13 * bt_t_13; + float qe_12 = q_t_1_13 * e_dg_13; + float kp_24 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_13) * kdg_0[2]); + float dqf_13 = pool_f32[ew_1 + 256 + d_lane]; + float dkf_13 = pool_f32[ew_1 + 1280 + d_lane]; + float dgf_13 = pool_f32[ew_1 + 2304 + d_lane]; + float dq_o_13 = dq2_13 + dqf_13; + float dk_o_13 = dk2b_13 + dkf_13 + dkt_2[2]; + float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_2[2], k_t_13, cr_12)) + dgf_13; + s_run_1[0] = s_run_1[0] + dg_o_13; +#else + int t_5 = 16 + 8 * th + 2; + float r_271[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_5 = r_271[0]; + unsigned int r_272[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_272[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_5 = __uint_as_float(r_272[0] << 16); + unsigned int r_273[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_273[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_5 = __uint_as_float(r_273[0] << 16); + float q_t_1_5 = ((t_5 >= clen) ? 0.0f : q_t_5); + float bt_t_5 = btv_2[2]; + float e_off_5 = approx_exp2(g_t_5 - gn_1); + float e_dg_5 = approx_exp2(g_t_5 - gc_2); + float dq2_5 = __fmaf_rn(qo_2[2], e_off_5, qdg_2[2] * e_dg_5); + float dk2_5 = __fmaf_rn(ko_2[2], e_off_5, kdg_2[2] * e_dg_5); + dbp_1[2] = dk2_5 * k_t_5; + float dk2b_5 = dk2_5 * bt_t_5; + float qe_4 = q_t_1_5 * e_dg_5; + float kp_16 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_16; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg_2[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_16 * e_dg_5) * kdg_2[2]); + float dqf_5 = pool_f32[ew_1 + 256 + d_lane]; + float dkf_5 = pool_f32[ew_1 + 1280 + d_lane]; + float dgf_5 = pool_f32[ew_1 + 2304 + d_lane]; + float dq_o_5 = dq2_5 + dqf_5; + float dk_o_5 = dk2b_5 + dkf_5 + dkt_2[2]; + float dg_o_5 = __fmaf_rn(q_t_1_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_2[2], k_t_5, cr_4)) + dgf_5; + s_run_1[0] = s_run_1[0] + dg_o_5; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 2)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_21 = t_21 * qrs; - if (t_21 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_13 = t_13 * qrs; + if (t_13 < clen) { +#else + int qoff_5 = t_5 * qrs; + if (t_5 < clen) { +#endif { - auto _gst_406 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_406)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_406) + (0)) = (__nv_bfloat16)dq_o_21; +#if __CUDA_ARCH__ == 1030 + auto _gst_424 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_424)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_424) + (0)) = (__nv_bfloat16)dq_o_13; +#else + auto _gst_410 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_410)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_5; +#endif } { - auto _gst_407 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_407)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_407) + (0)) = (__nv_bfloat16)dk_o_21; +#if __CUDA_ARCH__ == 1030 + auto _gst_425 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_425)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_425) + (0)) = (__nv_bfloat16)dk_o_13; +#else + auto _gst_411 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_411)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_5; +#endif } } - int t_22 = 16 + 8 * th + 1; - float r_316[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_22 = r_316[0]; - unsigned int r_317[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_22 = __uint_as_float(r_317[0] << 16); - unsigned int r_318[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_22 = __uint_as_float(r_318[0] << 16); - float q_t_1_22 = ((t_22 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1 + 128 + d_lane]; - float dkf_22 = pool_f32[ew_1 + 1152 + d_lane]; - float dgf_22 = pool_f32[ew_1 + 2176 + d_lane]; - float bt_t_22 = btv_1[1]; - float e_off_22 = approx_exp2(g_t_22 - gn_1); - float e_dg_22 = approx_exp2(g_t_22 - gc_2); - float dq2_22 = __fmaf_rn(qo_1[1], e_off_22, qdg_1[1] * e_dg_22); - float dk2_22 = __fmaf_rn(ko_1[1], e_off_22, kdg_1[1] * e_dg_22); - dbp_1[1] = dk2_22 * k_t_22; - float dk2b_22 = dk2_22 * bt_t_22; - float dq_o_22 = dq2_22 + dqf_22; - float dk_o_22 = dk2b_22 + dkf_22 + dkt_2[1]; - float dg_o_22 = __fmaf_rn(q_t_1_22, dq2_22, (dk2b_22 - dkt_2[1]) * k_t_22) + dgf_22; - s_run_1[0] = s_run_1[0] + dg_o_22; +#if __CUDA_ARCH__ == 1030 + int t_14 = 16 + 8 * th + 1; + float r_295[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_14 = r_295[0]; + unsigned int r_296[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_14 = __uint_as_float(r_296[0] << 16); + unsigned int r_297[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_14 = __uint_as_float(r_297[0] << 16); + float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); + float bt_t_14 = btv_0[1]; + float e_off_14 = approx_exp2(g_t_14 - gn_1); + float e_dg_14 = approx_exp2(g_t_14 - gc_2); + float dq2_14 = __fmaf_rn(qo_0[1], e_off_14, qdg_0[1] * e_dg_14); + float dk2_14 = __fmaf_rn(ko_0[1], e_off_14, kdg_0[1] * e_dg_14); + dbp_1[1] = dk2_14 * k_t_14; + float dk2b_14 = dk2_14 * bt_t_14; + float qe_13 = q_t_1_14 * e_dg_14; + float kp_25 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_14) * kdg_0[1]); + float dqf_14 = pool_f32[ew_1 + 128 + d_lane]; + float dkf_14 = pool_f32[ew_1 + 1152 + d_lane]; + float dgf_14 = pool_f32[ew_1 + 2176 + d_lane]; + float dq_o_14 = dq2_14 + dqf_14; + float dk_o_14 = dk2b_14 + dkf_14 + dkt_2[1]; + float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_2[1], k_t_14, cr_13)) + dgf_14; + s_run_1[0] = s_run_1[0] + dg_o_14; +#else + int t_6 = 16 + 8 * th + 1; + float r_274[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_274[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_6 = r_274[0]; + unsigned int r_275[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_6 = __uint_as_float(r_275[0] << 16); + unsigned int r_276[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_276[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_6 = __uint_as_float(r_276[0] << 16); + float q_t_1_6 = ((t_6 >= clen) ? 0.0f : q_t_6); + float bt_t_6 = btv_2[1]; + float e_off_6 = approx_exp2(g_t_6 - gn_1); + float e_dg_6 = approx_exp2(g_t_6 - gc_2); + float dq2_6 = __fmaf_rn(qo_2[1], e_off_6, qdg_2[1] * e_dg_6); + float dk2_6 = __fmaf_rn(ko_2[1], e_off_6, kdg_2[1] * e_dg_6); + dbp_1[1] = dk2_6 * k_t_6; + float dk2b_6 = dk2_6 * bt_t_6; + float qe_5 = q_t_1_6 * e_dg_6; + float kp_17 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_17; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg_2[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_17 * e_dg_6) * kdg_2[1]); + float dqf_6 = pool_f32[ew_1 + 128 + d_lane]; + float dkf_6 = pool_f32[ew_1 + 1152 + d_lane]; + float dgf_6 = pool_f32[ew_1 + 2176 + d_lane]; + float dq_o_6 = dq2_6 + dqf_6; + float dk_o_6 = dk2b_6 + dkf_6 + dkt_2[1]; + float dg_o_6 = __fmaf_rn(q_t_1_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_2[1], k_t_6, cr_5)) + dgf_6; + s_run_1[0] = s_run_1[0] + dg_o_6; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th + 1)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_22 = t_22 * qrs; - if (t_22 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_14 = t_14 * qrs; + if (t_14 < clen) { +#else + int qoff_6 = t_6 * qrs; + if (t_6 < clen) { +#endif { - auto _gst_408 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_408)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_408) + (0)) = (__nv_bfloat16)dq_o_22; +#if __CUDA_ARCH__ == 1030 + auto _gst_426 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_426)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_426) + (0)) = (__nv_bfloat16)dq_o_14; +#else + auto _gst_412 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_412)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_412) + (0)) = (__nv_bfloat16)dq_o_6; +#endif } { - auto _gst_409 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_409)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_409) + (0)) = (__nv_bfloat16)dk_o_22; +#if __CUDA_ARCH__ == 1030 + auto _gst_427 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_427)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_427) + (0)) = (__nv_bfloat16)dk_o_14; +#else + auto _gst_413 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_413)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_413) + (0)) = (__nv_bfloat16)dk_o_6; +#endif } } - int t_23 = 16 + 8 * th; - float r_319[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_23 = r_319[0]; - unsigned int r_320[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_23 = __uint_as_float(r_320[0] << 16); - unsigned int r_321[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_23 = __uint_as_float(r_321[0] << 16); - float q_t_1_23 = ((t_23 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1 + d_lane]; - float dkf_23 = pool_f32[ew_1 + 1024 + d_lane]; - float dgf_23 = pool_f32[ew_1 + 2048 + d_lane]; - float bt_t_23 = btv_1[0]; - float e_off_23 = approx_exp2(g_t_23 - gn_1); - float e_dg_23 = approx_exp2(g_t_23 - gc_2); - float dq2_23 = __fmaf_rn(qo_1[0], e_off_23, qdg_1[0] * e_dg_23); - float dk2_23 = __fmaf_rn(ko_1[0], e_off_23, kdg_1[0] * e_dg_23); - dbp_1[0] = dk2_23 * k_t_23; - float dk2b_23 = dk2_23 * bt_t_23; - float dq_o_23 = dq2_23 + dqf_23; - float dk_o_23 = dk2b_23 + dkf_23 + dkt_2[0]; - float dg_o_23 = __fmaf_rn(q_t_1_23, dq2_23, (dk2b_23 - dkt_2[0]) * k_t_23) + dgf_23; - s_run_1[0] = s_run_1[0] + dg_o_23; +#if __CUDA_ARCH__ == 1030 + int t_15 = 16 + 8 * th; + float r_298[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_15 = r_298[0]; + unsigned int r_299[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_15 = __uint_as_float(r_299[0] << 16); + unsigned int r_300[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_15 = __uint_as_float(r_300[0] << 16); + float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); + float bt_t_15 = btv_0[0]; + float e_off_15 = approx_exp2(g_t_15 - gn_1); + float e_dg_15 = approx_exp2(g_t_15 - gc_2); + float dq2_15 = __fmaf_rn(qo_0[0], e_off_15, qdg_0[0] * e_dg_15); + float dk2_15 = __fmaf_rn(ko_0[0], e_off_15, kdg_0[0] * e_dg_15); + dbp_1[0] = dk2_15 * k_t_15; + float dk2b_15 = dk2_15 * bt_t_15; + float qe_14 = q_t_1_15 * e_dg_15; + float kp_26 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_15) * kdg_0[0]); + float dqf_15 = pool_f32[ew_1 + d_lane]; + float dkf_15 = pool_f32[ew_1 + 1024 + d_lane]; + float dgf_15 = pool_f32[ew_1 + 2048 + d_lane]; + float dq_o_15 = dq2_15 + dqf_15; + float dk_o_15 = dk2b_15 + dkf_15 + dkt_2[0]; + float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_2[0], k_t_15, cr_14)) + dgf_15; + s_run_1[0] = s_run_1[0] + dg_o_15; +#else + int t_7 = 16 + 8 * th; + float r_277[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_277[0])) : "r"((gb8_1 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_7 = r_277[0]; + unsigned int r_278[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((kb8_1 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_7 = __uint_as_float(r_278[0] << 16); + unsigned int r_279[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_279[0])) : "r"((qbase_2 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_7 = __uint_as_float(r_279[0] << 16); + float q_t_1_7 = ((t_7 >= clen) ? 0.0f : q_t_7); + float bt_t_7 = btv_2[0]; + float e_off_7 = approx_exp2(g_t_7 - gn_1); + float e_dg_7 = approx_exp2(g_t_7 - gc_2); + float dq2_7 = __fmaf_rn(qo_2[0], e_off_7, qdg_2[0] * e_dg_7); + float dk2_7 = __fmaf_rn(ko_2[0], e_off_7, kdg_2[0] * e_dg_7); + dbp_1[0] = dk2_7 * k_t_7; + float dk2b_7 = dk2_7 * bt_t_7; + float qe_6 = q_t_1_7 * e_dg_7; + float kp_18 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_18; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg_2[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_18 * e_dg_7) * kdg_2[0]); + float dqf_7 = pool_f32[ew_1 + d_lane]; + float dkf_7 = pool_f32[ew_1 + 1024 + d_lane]; + float dgf_7 = pool_f32[ew_1 + 2048 + d_lane]; + float dq_o_7 = dq2_7 + dqf_7; + float dk_o_7 = dk2b_7 + dkf_7 + dkt_2[0]; + float dg_o_7 = __fmaf_rn(q_t_1_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_2[0], k_t_7, cr_6)) + dgf_7; + s_run_1[0] = s_run_1[0] + dg_o_7; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(96 + 8 * th)), "f"(s_run_1[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_23 = t_23 * qrs; - if (t_23 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_15 = t_15 * qrs; + if (t_15 < clen) { +#else + int qoff_7 = t_7 * qrs; + if (t_7 < clen) { +#endif { - auto _gst_410 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_410)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_410) + (0)) = (__nv_bfloat16)dq_o_23; +#if __CUDA_ARCH__ == 1030 + auto _gst_428 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_428)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_428) + (0)) = (__nv_bfloat16)dq_o_15; +#else + auto _gst_414 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_414)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_414) + (0)) = (__nv_bfloat16)dq_o_7; +#endif } { - auto _gst_411 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_411)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_411) + (0)) = (__nv_bfloat16)dk_o_23; +#if __CUDA_ARCH__ == 1030 + auto _gst_429 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_429)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_429) + (0)) = (__nv_bfloat16)dk_o_15; +#else + auto _gst_415 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_415)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_415) + (0)) = (__nv_bfloat16)dk_o_7; +#endif } } float tot_1 = s_run_1[0]; @@ -4860,39 +5859,83 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_412 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_412; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_430 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_430; +#else + float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_416; +#endif float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_413 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_413; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_431 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_431; +#else + float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_417; +#endif float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_414 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_414; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_432 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_432; +#else + float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_418; +#endif float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_415 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_415; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_433 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_433; +#else + float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_419; +#endif float v2_17[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_416 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_17[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_416; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_434 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_17[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_434; +#else + float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_17[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_420; +#endif float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_417 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_17[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_417; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_435 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_17[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_435; +#else + float _shfl_xor_sync_421 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_17[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_421; +#endif float send_19 = ((b2_1 == 0) ? v2_17[1] : v2_17[0]); - float _shfl_xor_sync_418 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_418; - float _shfl_xor_sync_419 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_419; - float _shfl_xor_sync_420 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_420; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_436 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_436; + float _shfl_xor_sync_437 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_437; + float _shfl_xor_sync_438 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_438; +#else + float _shfl_xor_sync_422 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_17[0] : v2_17[1]) + _shfl_xor_sync_422; + float _shfl_xor_sync_423 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_423; + float _shfl_xor_sync_424 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_424; +#endif asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_421 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_421), "f"(tot_1) : "memory"); +#if __CUDA_ARCH__ == 1030 + uint32_t _addr_439 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_439), "f"(tot_1) : "memory"); +#else + uint32_t _addr_425 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_425), "f"(tot_1) : "memory"); +#endif } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4924,132 +5967,280 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float dgr8_1[8]; float2 dyg2_8 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[0], sfx_1[1])); float2 rp_u_7 = add_f32x2(make_float2(raw_t1[0], raw_t1[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_422 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_440 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_426 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_422), "l"(*(const unsigned long long*)&rp_u_7)); - float _abs_425; - asm("abs.f32 %0, %1;" : "=f"(_abs_425) : "f"(rp_z_7.x)); - float _neg_424; - asm("neg.f32 %0, %1;" : "=f"(_neg_424) : "f"(_abs_425)); - float _exp_423 = __nv_expf(_neg_424); - float _abs_428; - asm("abs.f32 %0, %1;" : "=f"(_abs_428) : "f"(rp_z_7.y)); - float _neg_427; - asm("neg.f32 %0, %1;" : "=f"(_neg_427) : "f"(_abs_428)); - float _exp_426 = __nv_expf(_neg_427); - float2 rp_e_7 = make_float2(_exp_423, _exp_426); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_440), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_443; + asm("abs.f32 %0, %1;" : "=f"(_abs_443) : "f"(rp_z_7.x)); + float _neg_442; + asm("neg.f32 %0, %1;" : "=f"(_neg_442) : "f"(_abs_443)); + float _exp_441 = __nv_expf(_neg_442); + float _abs_446; + asm("abs.f32 %0, %1;" : "=f"(_abs_446) : "f"(rp_z_7.y)); + float _neg_445; + asm("neg.f32 %0, %1;" : "=f"(_neg_445) : "f"(_abs_446)); + float _exp_444 = __nv_expf(_neg_445); + float2 rp_e_7 = make_float2(_exp_441, _exp_444); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_426), "l"(*(const unsigned long long*)&rp_u_7)); + float _abs_429; + asm("abs.f32 %0, %1;" : "=f"(_abs_429) : "f"(rp_z_7.x)); + float _neg_428; + asm("neg.f32 %0, %1;" : "=f"(_neg_428) : "f"(_abs_429)); + float _exp_427 = __nv_expf(_neg_428); + float _abs_432; + asm("abs.f32 %0, %1;" : "=f"(_abs_432) : "f"(rp_z_7.y)); + float _neg_431; + asm("neg.f32 %0, %1;" : "=f"(_neg_431) : "f"(_abs_432)); + float _exp_430 = __nv_expf(_neg_431); + float2 rp_e_7 = make_float2(_exp_427, _exp_430); +#endif float2 rp_dn_7 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_7); float2 rp_den_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_7) : "l"(*(const unsigned long long*)&rp_dn_7), "l"(*(const unsigned long long*)&rp_dn_7)); - float _rcp_429 = approx_rcp(rp_den_7.x); - float _rcp_430 = approx_rcp(rp_den_7.y); - float2 _f32x2_mul_b_431 = make_float2(_rcp_429, _rcp_430); +#if __CUDA_ARCH__ == 1030 + float _rcp_447 = approx_rcp(rp_den_7.x); + float _rcp_448 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_449 = make_float2(_rcp_447, _rcp_448); +#else + float _rcp_433 = approx_rcp(rp_den_7.x); + float _rcp_434 = approx_rcp(rp_den_7.y); + float2 _f32x2_mul_b_435 = make_float2(_rcp_433, _rcp_434); +#endif float2 rp_sp_7; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_431)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_449)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_7) : "l"(*(const unsigned long long*)&rp_e_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_435)); +#endif float2 rp_t_7; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_7) : "l"(*(const unsigned long long*)&dyg2_8), "l"(*(const unsigned long long*)&rp_sp_7)); - float2 _f32x2_mul_b_432 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_450 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_436 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_432)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_450)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_7) : "l"(*(const unsigned long long*)&rp_t_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_436)); +#endif u8_1[0] = rp_u_7.x; u8_1[1] = rp_u_7.y; dgr8_1[0] = rp_dgr_7.x; dgr8_1[1] = rp_dgr_7.y; float2 dyg2_9 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[2], sfx_1[3])); float2 rp_u_8 = add_f32x2(make_float2(raw_t1[2], raw_t1[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_433 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_451 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_437 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_433), "l"(*(const unsigned long long*)&rp_u_8)); - float _abs_436; - asm("abs.f32 %0, %1;" : "=f"(_abs_436) : "f"(rp_z_8.x)); - float _neg_435; - asm("neg.f32 %0, %1;" : "=f"(_neg_435) : "f"(_abs_436)); - float _exp_434 = __nv_expf(_neg_435); - float _abs_439; - asm("abs.f32 %0, %1;" : "=f"(_abs_439) : "f"(rp_z_8.y)); - float _neg_438; - asm("neg.f32 %0, %1;" : "=f"(_neg_438) : "f"(_abs_439)); - float _exp_437 = __nv_expf(_neg_438); - float2 rp_e_8 = make_float2(_exp_434, _exp_437); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_451), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_454; + asm("abs.f32 %0, %1;" : "=f"(_abs_454) : "f"(rp_z_8.x)); + float _neg_453; + asm("neg.f32 %0, %1;" : "=f"(_neg_453) : "f"(_abs_454)); + float _exp_452 = __nv_expf(_neg_453); + float _abs_457; + asm("abs.f32 %0, %1;" : "=f"(_abs_457) : "f"(rp_z_8.y)); + float _neg_456; + asm("neg.f32 %0, %1;" : "=f"(_neg_456) : "f"(_abs_457)); + float _exp_455 = __nv_expf(_neg_456); + float2 rp_e_8 = make_float2(_exp_452, _exp_455); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_437), "l"(*(const unsigned long long*)&rp_u_8)); + float _abs_440; + asm("abs.f32 %0, %1;" : "=f"(_abs_440) : "f"(rp_z_8.x)); + float _neg_439; + asm("neg.f32 %0, %1;" : "=f"(_neg_439) : "f"(_abs_440)); + float _exp_438 = __nv_expf(_neg_439); + float _abs_443; + asm("abs.f32 %0, %1;" : "=f"(_abs_443) : "f"(rp_z_8.y)); + float _neg_442; + asm("neg.f32 %0, %1;" : "=f"(_neg_442) : "f"(_abs_443)); + float _exp_441 = __nv_expf(_neg_442); + float2 rp_e_8 = make_float2(_exp_438, _exp_441); +#endif float2 rp_dn_8 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_8); float2 rp_den_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_8) : "l"(*(const unsigned long long*)&rp_dn_8), "l"(*(const unsigned long long*)&rp_dn_8)); - float _rcp_440 = approx_rcp(rp_den_8.x); - float _rcp_441 = approx_rcp(rp_den_8.y); - float2 _f32x2_mul_b_442 = make_float2(_rcp_440, _rcp_441); +#if __CUDA_ARCH__ == 1030 + float _rcp_458 = approx_rcp(rp_den_8.x); + float _rcp_459 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_460 = make_float2(_rcp_458, _rcp_459); +#else + float _rcp_444 = approx_rcp(rp_den_8.x); + float _rcp_445 = approx_rcp(rp_den_8.y); + float2 _f32x2_mul_b_446 = make_float2(_rcp_444, _rcp_445); +#endif float2 rp_sp_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_442)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_460)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_8) : "l"(*(const unsigned long long*)&rp_e_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_446)); +#endif float2 rp_t_8; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_8) : "l"(*(const unsigned long long*)&dyg2_9), "l"(*(const unsigned long long*)&rp_sp_8)); - float2 _f32x2_mul_b_443 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_461 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_447 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_443)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_461)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_8) : "l"(*(const unsigned long long*)&rp_t_8), "l"(*(const unsigned long long*)&_f32x2_mul_b_447)); +#endif u8_1[2] = rp_u_8.x; u8_1[3] = rp_u_8.y; dgr8_1[2] = rp_dgr_8.x; dgr8_1[3] = rp_dgr_8.y; float2 dyg2_10 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[4], sfx_1[5])); float2 rp_u_9 = add_f32x2(make_float2(raw_t1[4], raw_t1[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_444 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_462 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_448 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_444), "l"(*(const unsigned long long*)&rp_u_9)); - float _abs_447; - asm("abs.f32 %0, %1;" : "=f"(_abs_447) : "f"(rp_z_9.x)); - float _neg_446; - asm("neg.f32 %0, %1;" : "=f"(_neg_446) : "f"(_abs_447)); - float _exp_445 = __nv_expf(_neg_446); - float _abs_450; - asm("abs.f32 %0, %1;" : "=f"(_abs_450) : "f"(rp_z_9.y)); - float _neg_449; - asm("neg.f32 %0, %1;" : "=f"(_neg_449) : "f"(_abs_450)); - float _exp_448 = __nv_expf(_neg_449); - float2 rp_e_9 = make_float2(_exp_445, _exp_448); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_462), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_465; + asm("abs.f32 %0, %1;" : "=f"(_abs_465) : "f"(rp_z_9.x)); + float _neg_464; + asm("neg.f32 %0, %1;" : "=f"(_neg_464) : "f"(_abs_465)); + float _exp_463 = __nv_expf(_neg_464); + float _abs_468; + asm("abs.f32 %0, %1;" : "=f"(_abs_468) : "f"(rp_z_9.y)); + float _neg_467; + asm("neg.f32 %0, %1;" : "=f"(_neg_467) : "f"(_abs_468)); + float _exp_466 = __nv_expf(_neg_467); + float2 rp_e_9 = make_float2(_exp_463, _exp_466); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_448), "l"(*(const unsigned long long*)&rp_u_9)); + float _abs_451; + asm("abs.f32 %0, %1;" : "=f"(_abs_451) : "f"(rp_z_9.x)); + float _neg_450; + asm("neg.f32 %0, %1;" : "=f"(_neg_450) : "f"(_abs_451)); + float _exp_449 = __nv_expf(_neg_450); + float _abs_454; + asm("abs.f32 %0, %1;" : "=f"(_abs_454) : "f"(rp_z_9.y)); + float _neg_453; + asm("neg.f32 %0, %1;" : "=f"(_neg_453) : "f"(_abs_454)); + float _exp_452 = __nv_expf(_neg_453); + float2 rp_e_9 = make_float2(_exp_449, _exp_452); +#endif float2 rp_dn_9 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_9); float2 rp_den_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_9) : "l"(*(const unsigned long long*)&rp_dn_9), "l"(*(const unsigned long long*)&rp_dn_9)); - float _rcp_451 = approx_rcp(rp_den_9.x); - float _rcp_452 = approx_rcp(rp_den_9.y); - float2 _f32x2_mul_b_453 = make_float2(_rcp_451, _rcp_452); +#if __CUDA_ARCH__ == 1030 + float _rcp_469 = approx_rcp(rp_den_9.x); + float _rcp_470 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_471 = make_float2(_rcp_469, _rcp_470); +#else + float _rcp_455 = approx_rcp(rp_den_9.x); + float _rcp_456 = approx_rcp(rp_den_9.y); + float2 _f32x2_mul_b_457 = make_float2(_rcp_455, _rcp_456); +#endif float2 rp_sp_9; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_453)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_471)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_9) : "l"(*(const unsigned long long*)&rp_e_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_457)); +#endif float2 rp_t_9; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_9) : "l"(*(const unsigned long long*)&dyg2_10), "l"(*(const unsigned long long*)&rp_sp_9)); - float2 _f32x2_mul_b_454 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_472 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_458 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_454)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_472)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_9) : "l"(*(const unsigned long long*)&rp_t_9), "l"(*(const unsigned long long*)&_f32x2_mul_b_458)); +#endif u8_1[4] = rp_u_9.x; u8_1[5] = rp_u_9.y; dgr8_1[4] = rp_dgr_9.x; dgr8_1[5] = rp_dgr_9.y; float2 dyg2_11 = add_f32x2(make_float2(offset_1, offset_1), make_float2(sfx_1[6], sfx_1[7])); float2 rp_u_10 = add_f32x2(make_float2(raw_t1[6], raw_t1[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_455 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_473 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_459 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_455), "l"(*(const unsigned long long*)&rp_u_10)); - float _abs_458; - asm("abs.f32 %0, %1;" : "=f"(_abs_458) : "f"(rp_z_10.x)); - float _neg_457; - asm("neg.f32 %0, %1;" : "=f"(_neg_457) : "f"(_abs_458)); - float _exp_456 = __nv_expf(_neg_457); - float _abs_461; - asm("abs.f32 %0, %1;" : "=f"(_abs_461) : "f"(rp_z_10.y)); - float _neg_460; - asm("neg.f32 %0, %1;" : "=f"(_neg_460) : "f"(_abs_461)); - float _exp_459 = __nv_expf(_neg_460); - float2 rp_e_10 = make_float2(_exp_456, _exp_459); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_473), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_476; + asm("abs.f32 %0, %1;" : "=f"(_abs_476) : "f"(rp_z_10.x)); + float _neg_475; + asm("neg.f32 %0, %1;" : "=f"(_neg_475) : "f"(_abs_476)); + float _exp_474 = __nv_expf(_neg_475); + float _abs_479; + asm("abs.f32 %0, %1;" : "=f"(_abs_479) : "f"(rp_z_10.y)); + float _neg_478; + asm("neg.f32 %0, %1;" : "=f"(_neg_478) : "f"(_abs_479)); + float _exp_477 = __nv_expf(_neg_478); + float2 rp_e_10 = make_float2(_exp_474, _exp_477); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_459), "l"(*(const unsigned long long*)&rp_u_10)); + float _abs_462; + asm("abs.f32 %0, %1;" : "=f"(_abs_462) : "f"(rp_z_10.x)); + float _neg_461; + asm("neg.f32 %0, %1;" : "=f"(_neg_461) : "f"(_abs_462)); + float _exp_460 = __nv_expf(_neg_461); + float _abs_465; + asm("abs.f32 %0, %1;" : "=f"(_abs_465) : "f"(rp_z_10.y)); + float _neg_464; + asm("neg.f32 %0, %1;" : "=f"(_neg_464) : "f"(_abs_465)); + float _exp_463 = __nv_expf(_neg_464); + float2 rp_e_10 = make_float2(_exp_460, _exp_463); +#endif float2 rp_dn_10 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_10); float2 rp_den_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_10) : "l"(*(const unsigned long long*)&rp_dn_10), "l"(*(const unsigned long long*)&rp_dn_10)); - float _rcp_462 = approx_rcp(rp_den_10.x); - float _rcp_463 = approx_rcp(rp_den_10.y); - float2 _f32x2_mul_b_464 = make_float2(_rcp_462, _rcp_463); +#if __CUDA_ARCH__ == 1030 + float _rcp_480 = approx_rcp(rp_den_10.x); + float _rcp_481 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_482 = make_float2(_rcp_480, _rcp_481); +#else + float _rcp_466 = approx_rcp(rp_den_10.x); + float _rcp_467 = approx_rcp(rp_den_10.y); + float2 _f32x2_mul_b_468 = make_float2(_rcp_466, _rcp_467); +#endif float2 rp_sp_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_464)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_482)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_10) : "l"(*(const unsigned long long*)&rp_e_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_468)); +#endif float2 rp_t_10; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_10) : "l"(*(const unsigned long long*)&dyg2_11), "l"(*(const unsigned long long*)&rp_sp_10)); - float2 _f32x2_mul_b_465 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_483 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_469 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_465)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_483)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_10) : "l"(*(const unsigned long long*)&rp_t_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_469)); +#endif u8_1[6] = rp_u_10.x; u8_1[7] = rp_u_10.y; dgr8_1[6] = rp_dgr_10.x; @@ -5064,54 +6255,102 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d } unsigned int pv_8 = __shfl_xor_sync(0xFFFFFFFF, pw_1[0], 1); unsigned int pq_8[1]; - uint32_t _prmt_466; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_466) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); - pq_8[0] = _prmt_466; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_484; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_484) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_484; +#else + uint32_t _prmt_470; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[0]), "r"(pv_8), "r"(sel_1)); + pq_8[0] = _prmt_470; +#endif int t_q_8 = 16 + 8 * th + odd_1; if (t_q_8 < clen) { { - auto _gst_467 = draw_base + (t_q_8 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_467)); - *(reinterpret_cast(_gst_467) + (0)) = pq_8[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_485 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_485)); + *(reinterpret_cast(_gst_485) + (0)) = pq_8[0]; +#else + auto _gst_471 = draw_base + (t_q_8 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_471)); + *(reinterpret_cast(_gst_471) + (0)) = pq_8[0]; +#endif } } unsigned int pv_9 = __shfl_xor_sync(0xFFFFFFFF, pw_1[1], 1); unsigned int pq_9[1]; - uint32_t _prmt_468; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_468) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); - pq_9[0] = _prmt_468; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_486; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_486) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_486; +#else + uint32_t _prmt_472; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[1]), "r"(pv_9), "r"(sel_1)); + pq_9[0] = _prmt_472; +#endif int t_q_9 = 16 + 8 * th + 2 + odd_1; if (t_q_9 < clen) { { - auto _gst_469 = draw_base + (t_q_9 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_469)); - *(reinterpret_cast(_gst_469) + (0)) = pq_9[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_487 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_487)); + *(reinterpret_cast(_gst_487) + (0)) = pq_9[0]; +#else + auto _gst_473 = draw_base + (t_q_9 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_473)); + *(reinterpret_cast(_gst_473) + (0)) = pq_9[0]; +#endif } } unsigned int pv_10 = __shfl_xor_sync(0xFFFFFFFF, pw_1[2], 1); unsigned int pq_10[1]; - uint32_t _prmt_470; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_470) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); - pq_10[0] = _prmt_470; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_488; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_488) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_488; +#else + uint32_t _prmt_474; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_474) : "r"(pw_1[2]), "r"(pv_10), "r"(sel_1)); + pq_10[0] = _prmt_474; +#endif int t_q_10 = 16 + 8 * th + 4 + odd_1; if (t_q_10 < clen) { { - auto _gst_471 = draw_base + (t_q_10 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_471)); - *(reinterpret_cast(_gst_471) + (0)) = pq_10[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_489 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_489)); + *(reinterpret_cast(_gst_489) + (0)) = pq_10[0]; +#else + auto _gst_475 = draw_base + (t_q_10 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_475)); + *(reinterpret_cast(_gst_475) + (0)) = pq_10[0]; +#endif } } unsigned int pv_11 = __shfl_xor_sync(0xFFFFFFFF, pw_1[3], 1); unsigned int pq_11[1]; - uint32_t _prmt_472; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_472) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); - pq_11[0] = _prmt_472; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_490; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_490) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_490; +#else + uint32_t _prmt_476; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_476) : "r"(pw_1[3]), "r"(pv_11), "r"(sel_1)); + pq_11[0] = _prmt_476; +#endif int t_q_11 = 16 + 8 * th + 6 + odd_1; if (t_q_11 < clen) { { - auto _gst_473 = draw_base + (t_q_11 * rs_g - odd_1); - __builtin_assume(__isGlobal(_gst_473)); - *(reinterpret_cast(_gst_473) + (0)) = pq_11[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_491 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_491)); + *(reinterpret_cast(_gst_491) + (0)) = pq_11[0]; +#else + auto _gst_477 = draw_base + (t_q_11 * rs_g - odd_1); + __builtin_assume(__isGlobal(_gst_477)); + *(reinterpret_cast(_gst_477) + (0)) = pq_11[0]; +#endif } } int t_a_16 = 16 + 8 * th; @@ -5176,47 +6415,99 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_1) : "f"(pa01_1), "f"(pa23_1)); float nm_t_3; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_3) : "f"(s_a_1), "f"(pa_1)); - float _sub_474; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_474) : "f"(s_a_1), "f"(nm_t_3)); +#if __CUDA_ARCH__ == 1030 + float _sub_492; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_492) : "f"(s_a_1), "f"(nm_t_3)); +#else + float _sub_478; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_478) : "f"(s_a_1), "f"(nm_t_3)); +#endif float nm_es_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_474), "f"(pa_1)); - float _sub_475; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_475) : "f"(pa_1), "f"(nm_t_3)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_492), "f"(pa_1)); + float _sub_493; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_493) : "f"(pa_1), "f"(nm_t_3)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_3) : "f"(_sub_478), "f"(pa_1)); + float _sub_479; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_479) : "f"(pa_1), "f"(nm_t_3)); +#endif float nm_ex_3; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_475), "f"(s_a_1)); - float _abs_477; - asm("abs.f32 %0, %1;" : "=f"(_abs_477) : "f"(s_a_1)); - float _abs_478; - asm("abs.f32 %0, %1;" : "=f"(_abs_478) : "f"(pa_1)); - uint32_t _setp_476_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_476_bits) : "f"(_abs_477), "f"(_abs_478)); - bool _setp_476 = _setp_476_bits != 0; - float nm_err_3 = ((_setp_476) ? nm_es_3 : nm_ex_3); - float _add_479; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_479) : "f"(c_a_1), "f"(nm_err_3)); - c_a_1 = _add_479; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_493), "f"(s_a_1)); + float _abs_495; + asm("abs.f32 %0, %1;" : "=f"(_abs_495) : "f"(s_a_1)); + float _abs_496; + asm("abs.f32 %0, %1;" : "=f"(_abs_496) : "f"(pa_1)); + uint32_t _setp_494_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_494_bits) : "f"(_abs_495), "f"(_abs_496)); + bool _setp_494 = _setp_494_bits != 0; + float nm_err_3 = ((_setp_494) ? nm_es_3 : nm_ex_3); + float _add_497; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_497) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_497; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_3) : "f"(_sub_479), "f"(s_a_1)); + float _abs_481; + asm("abs.f32 %0, %1;" : "=f"(_abs_481) : "f"(s_a_1)); + float _abs_482; + asm("abs.f32 %0, %1;" : "=f"(_abs_482) : "f"(pa_1)); + uint32_t _setp_480_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_480_bits) : "f"(_abs_481), "f"(_abs_482)); + bool _setp_480 = _setp_480_bits != 0; + float nm_err_3 = ((_setp_480) ? nm_es_3 : nm_ex_3); + float _add_483; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_483) : "f"(c_a_1), "f"(nm_err_3)); + c_a_1 = _add_483; +#endif s_a_1 = nm_t_3; float nm_t_4; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_4) : "f"(s_b_1), "f"(pb_1)); - float _sub_480; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_480) : "f"(s_b_1), "f"(nm_t_4)); +#if __CUDA_ARCH__ == 1030 + float _sub_498; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_498) : "f"(s_b_1), "f"(nm_t_4)); +#else + float _sub_484; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_484) : "f"(s_b_1), "f"(nm_t_4)); +#endif float nm_es_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_480), "f"(pb_1)); - float _sub_481; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_481) : "f"(pb_1), "f"(nm_t_4)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_498), "f"(pb_1)); + float _sub_499; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_499) : "f"(pb_1), "f"(nm_t_4)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_4) : "f"(_sub_484), "f"(pb_1)); + float _sub_485; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_485) : "f"(pb_1), "f"(nm_t_4)); +#endif float nm_ex_4; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_481), "f"(s_b_1)); - float _abs_483; - asm("abs.f32 %0, %1;" : "=f"(_abs_483) : "f"(s_b_1)); - float _abs_484; - asm("abs.f32 %0, %1;" : "=f"(_abs_484) : "f"(pb_1)); - uint32_t _setp_482_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_482_bits) : "f"(_abs_483), "f"(_abs_484)); - bool _setp_482 = _setp_482_bits != 0; - float nm_err_4 = ((_setp_482) ? nm_es_4 : nm_ex_4); - float _add_485; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_485) : "f"(c_b_1), "f"(nm_err_4)); - c_b_1 = _add_485; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_499), "f"(s_b_1)); + float _abs_501; + asm("abs.f32 %0, %1;" : "=f"(_abs_501) : "f"(s_b_1)); + float _abs_502; + asm("abs.f32 %0, %1;" : "=f"(_abs_502) : "f"(pb_1)); + uint32_t _setp_500_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_500_bits) : "f"(_abs_501), "f"(_abs_502)); + bool _setp_500 = _setp_500_bits != 0; + float nm_err_4 = ((_setp_500) ? nm_es_4 : nm_ex_4); + float _add_503; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_503) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_503; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_4) : "f"(_sub_485), "f"(s_b_1)); + float _abs_487; + asm("abs.f32 %0, %1;" : "=f"(_abs_487) : "f"(s_b_1)); + float _abs_488; + asm("abs.f32 %0, %1;" : "=f"(_abs_488) : "f"(pb_1)); + uint32_t _setp_486_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_486_bits) : "f"(_abs_487), "f"(_abs_488)); + bool _setp_486 = _setp_486_bits != 0; + float nm_err_4 = ((_setp_486) ? nm_es_4 : nm_ex_4); + float _add_489; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_489) : "f"(c_b_1), "f"(nm_err_4)); + c_b_1 = _add_489; +#endif s_b_1 = nm_t_4; gacc[0] = s_a_1; gacc[1] = c_a_1; @@ -5224,8 +6515,13 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d gacc[3] = c_b_1; off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_486 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_486), "f"(off_acc[0]) : "memory"); +#if __CUDA_ARCH__ == 1030 + uint32_t _addr_504 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_504), "f"(off_acc[0]) : "memory"); +#else + uint32_t _addr_490 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_490), "f"(off_acc[0]) : "memory"); +#endif } tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th), gacc); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); @@ -5249,9 +6545,15 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d raw_t0[7] = (float)g_raw[rb_2 + (long long)t_c_31 * g_raw_row_stride]; mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); - float r_322[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float gn_2 = r_322[0]; +#if __CUDA_ARCH__ == 1030 + float r_301[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_2 = r_301[0]; +#else + float r_280[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_280[0])) : "r"((gk_p_addr + (unsigned int)(qd * 8192) + (unsigned int)(lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float gn_2 = r_280[0]; +#endif float gc_3 = gn_2 - 21.640425f; int eb_2 = 98304 + th * -32768; int ew_2 = eb_2 >> 2; @@ -5260,364 +6562,823 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int gb8_2 = pool_f32_addr + (unsigned int)(qd * 8192 + sb8_2 * 128); int kb8_2 = pool_f32_addr + (unsigned int)(32768 + dh_lane * 8192 + sb8_2 * 128); float dbp_2[8]; - float qdg_2[8]; - tmem_ld_x8(&qdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); - float kdg_2[8]; - tmem_ld_x8(&kdg_2[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); +#if __CUDA_ARCH__ == 1030 + float qdg_1_1[8]; + tmem_ld_x8(&qdg_1_1[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); + float kdg_1_1[8]; + tmem_ld_x8(&kdg_1_1[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); +#else + float qdg_0[8]; + tmem_ld_x8(&qdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)); + float kdg_0[8]; + tmem_ld_x8(&kdg_0[0], taddr + (unsigned int)row_base + (unsigned int)(144 + 8 * th)); +#endif float dkt_3[8]; tmem_ld_x8(&dkt_3[0], taddr + (unsigned int)row_base + (unsigned int)(8 * th)); - float btv_2[8]; +#if __CUDA_ARCH__ == 1030 + float btv_1_1[8]; +#else + float btv_0[8]; +#endif asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_2[0])), "=r"(*reinterpret_cast(&btv_2[(0) + 1])), "=r"(*reinterpret_cast(&btv_2[(0) + 2])), "=r"(*reinterpret_cast(&btv_2[(0) + 3])) +#if __CUDA_ARCH__ == 1030 + : "=r"(*reinterpret_cast(&btv_1_1[0])), "=r"(*reinterpret_cast(&btv_1_1[(0) + 1])), "=r"(*reinterpret_cast(&btv_1_1[(0) + 2])), "=r"(*reinterpret_cast(&btv_1_1[(0) + 3])) +#else + : "=r"(*reinterpret_cast(&btv_0[0])), "=r"(*reinterpret_cast(&btv_0[(0) + 1])), "=r"(*reinterpret_cast(&btv_0[(0) + 2])), "=r"(*reinterpret_cast(&btv_0[(0) + 3])) +#endif : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th)))); asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&btv_2[4])), "=r"(*reinterpret_cast(&btv_2[(4) + 1])), "=r"(*reinterpret_cast(&btv_2[(4) + 2])), "=r"(*reinterpret_cast(&btv_2[(4) + 3])) +#if __CUDA_ARCH__ == 1030 + : "=r"(*reinterpret_cast(&btv_1_1[4])), "=r"(*reinterpret_cast(&btv_1_1[(4) + 1])), "=r"(*reinterpret_cast(&btv_1_1[(4) + 2])), "=r"(*reinterpret_cast(&btv_1_1[(4) + 3])) +#else + : "=r"(*reinterpret_cast(&btv_0[4])), "=r"(*reinterpret_cast(&btv_0[(4) + 1])), "=r"(*reinterpret_cast(&btv_0[(4) + 2])), "=r"(*reinterpret_cast(&btv_0[(4) + 3])) +#endif : "r"(pool_f32_addr + (unsigned int)(114688 + 4 * (8 * th) + 16))); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float s_run_2[1]; s_run_2[0] = 0.0f; - int t_24 = 8 * th + 7; - float r_323[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_24 = r_323[0]; - unsigned int r_324[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_24 = __uint_as_float(r_324[0] << 16); - unsigned int r_325[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_24 = __uint_as_float(r_325[0] << 16); - float q_t_1_24 = ((t_24 >= clen) ? 0.0f : q_t_24); - float dqf_24 = pool_f32[ew_2 + 896 + d_lane]; - float dkf_24 = pool_f32[ew_2 + 1920 + d_lane]; - float dgf_24 = pool_f32[ew_2 + 2944 + d_lane]; - float bt_t_24 = btv_2[7]; - float e_off_24 = approx_exp2(g_t_24 - gn_2); - float e_dg_24 = approx_exp2(g_t_24 - gc_3); - float dq2_24 = qdg_2[7] * e_dg_24; - float dk2_24 = kdg_2[7] * e_dg_24; - dbp_2[7] = dk2_24 * k_t_24; - float dk2b_24 = dk2_24 * bt_t_24; - float dq_o_24 = dq2_24 + dqf_24; - float dk_o_24 = dk2b_24 + dkf_24 + dkt_3[7]; - float dg_o_24 = __fmaf_rn(q_t_1_24, dq2_24, (dk2b_24 - dkt_3[7]) * k_t_24) + dgf_24; - s_run_2[0] = s_run_2[0] + dg_o_24; +#if __CUDA_ARCH__ == 1030 + int t_16 = 8 * th + 7; + float r_302[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_16 = r_302[0]; + unsigned int r_303[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_16 = __uint_as_float(r_303[0] << 16); + unsigned int r_304[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_16 = __uint_as_float(r_304[0] << 16); + float q_t_1_16 = ((t_16 >= clen) ? 0.0f : q_t_16); + float bt_t_16 = btv_1_1[7]; + float e_off_16 = approx_exp2(g_t_16 - gn_2); + float e_dg_16 = approx_exp2(g_t_16 - gc_3); + float dq2_16 = qdg_1_1[7] * e_dg_16; + float dk2_16 = kdg_1_1[7] * e_dg_16; + dbp_2[7] = dk2_16 * k_t_16; + float dk2b_16 = dk2_16 * bt_t_16; + float qe_15 = q_t_1_16 * e_dg_16; + float kp_27 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_27; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1_1[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_27 * e_dg_16) * kdg_1_1[7]); + float dqf_16 = pool_f32[ew_2 + 896 + d_lane]; + float dkf_16 = pool_f32[ew_2 + 1920 + d_lane]; + float dgf_16 = pool_f32[ew_2 + 2944 + d_lane]; + float dq_o_16 = dq2_16 + dqf_16; + float dk_o_16 = dk2b_16 + dkf_16 + dkt_3[7]; + float dg_o_16 = __fmaf_rn(q_t_1_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_3[7], k_t_16, cr_15)) + dgf_16; + s_run_2[0] = s_run_2[0] + dg_o_16; +#else + int t_8 = 8 * th + 7; + float r_281[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 896 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_8 = r_281[0]; + unsigned int r_282[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_282[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_8 = __uint_as_float(r_282[0] << 16); + unsigned int r_283[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_283[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_8 = __uint_as_float(r_283[0] << 16); + float q_t_1_8 = ((t_8 >= clen) ? 0.0f : q_t_8); + float bt_t_8 = btv_0[7]; + float e_off_8 = approx_exp2(g_t_8 - gn_2); + float e_dg_8 = approx_exp2(g_t_8 - gc_3); + float dq2_8 = qdg_0[7] * e_dg_8; + float dk2_8 = kdg_0[7] * e_dg_8; + dbp_2[7] = dk2_8 * k_t_8; + float dk2b_8 = dk2_8 * bt_t_8; + float qe_7 = q_t_1_8 * e_dg_8; + float kp_19 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_19; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_19 * e_dg_8) * kdg_0[7]); + float dqf_8 = pool_f32[ew_2 + 896 + d_lane]; + float dkf_8 = pool_f32[ew_2 + 1920 + d_lane]; + float dgf_8 = pool_f32[ew_2 + 2944 + d_lane]; + float dq_o_8 = dq2_8 + dqf_8; + float dk_o_8 = dk2b_8 + dkf_8 + dkt_3[7]; + float dg_o_8 = __fmaf_rn(q_t_1_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_3[7], k_t_8, cr_7)) + dgf_8; + s_run_2[0] = s_run_2[0] + dg_o_8; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 7)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_24 = t_24 * qrs; - if (t_24 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_16 = t_16 * qrs; + if (t_16 < clen) { +#else + int qoff_8 = t_8 * qrs; + if (t_8 < clen) { +#endif { - auto _gst_487 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_487)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_487) + (0)) = (__nv_bfloat16)dq_o_24; +#if __CUDA_ARCH__ == 1030 + auto _gst_505 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_505)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_505) + (0)) = (__nv_bfloat16)dq_o_16; +#else + auto _gst_491 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_491)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_8; +#endif } { - auto _gst_488 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_488)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_488) + (0)) = (__nv_bfloat16)dk_o_24; +#if __CUDA_ARCH__ == 1030 + auto _gst_506 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_506)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_506) + (0)) = (__nv_bfloat16)dk_o_16; +#else + auto _gst_492 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_492)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_8; +#endif } } - int t_25 = 8 * th + 6; - float r_326[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_326[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_25 = r_326[0]; - unsigned int r_327[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_327[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_25 = __uint_as_float(r_327[0] << 16); - unsigned int r_328[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_25 = __uint_as_float(r_328[0] << 16); - float q_t_1_25 = ((t_25 >= clen) ? 0.0f : q_t_25); - float dqf_25 = pool_f32[ew_2 + 768 + d_lane]; - float dkf_25 = pool_f32[ew_2 + 1792 + d_lane]; - float dgf_25 = pool_f32[ew_2 + 2816 + d_lane]; - float bt_t_25 = btv_2[6]; - float e_off_25 = approx_exp2(g_t_25 - gn_2); - float e_dg_25 = approx_exp2(g_t_25 - gc_3); - float dq2_25 = qdg_2[6] * e_dg_25; - float dk2_25 = kdg_2[6] * e_dg_25; - dbp_2[6] = dk2_25 * k_t_25; - float dk2b_25 = dk2_25 * bt_t_25; - float dq_o_25 = dq2_25 + dqf_25; - float dk_o_25 = dk2b_25 + dkf_25 + dkt_3[6]; - float dg_o_25 = __fmaf_rn(q_t_1_25, dq2_25, (dk2b_25 - dkt_3[6]) * k_t_25) + dgf_25; - s_run_2[0] = s_run_2[0] + dg_o_25; +#if __CUDA_ARCH__ == 1030 + int t_17 = 8 * th + 6; + float r_305[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_305[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_17 = r_305[0]; + unsigned int r_306[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_17 = __uint_as_float(r_306[0] << 16); + unsigned int r_307[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_307[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_17 = __uint_as_float(r_307[0] << 16); + float q_t_1_17 = ((t_17 >= clen) ? 0.0f : q_t_17); + float bt_t_17 = btv_1_1[6]; + float e_off_17 = approx_exp2(g_t_17 - gn_2); + float e_dg_17 = approx_exp2(g_t_17 - gc_3); + float dq2_17 = qdg_1_1[6] * e_dg_17; + float dk2_17 = kdg_1_1[6] * e_dg_17; + dbp_2[6] = dk2_17 * k_t_17; + float dk2b_17 = dk2_17 * bt_t_17; + float qe_16 = q_t_1_17 * e_dg_17; + float kp_28 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_28; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1_1[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_28 * e_dg_17) * kdg_1_1[6]); + float dqf_17 = pool_f32[ew_2 + 768 + d_lane]; + float dkf_17 = pool_f32[ew_2 + 1792 + d_lane]; + float dgf_17 = pool_f32[ew_2 + 2816 + d_lane]; + float dq_o_17 = dq2_17 + dqf_17; + float dk_o_17 = dk2b_17 + dkf_17 + dkt_3[6]; + float dg_o_17 = __fmaf_rn(q_t_1_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_3[6], k_t_17, cr_16)) + dgf_17; + s_run_2[0] = s_run_2[0] + dg_o_17; +#else + int t_9 = 8 * th + 6; + float r_284[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 768 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_9 = r_284[0]; + unsigned int r_285[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_285[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_9 = __uint_as_float(r_285[0] << 16); + unsigned int r_286[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_286[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_9 = __uint_as_float(r_286[0] << 16); + float q_t_1_9 = ((t_9 >= clen) ? 0.0f : q_t_9); + float bt_t_9 = btv_0[6]; + float e_off_9 = approx_exp2(g_t_9 - gn_2); + float e_dg_9 = approx_exp2(g_t_9 - gc_3); + float dq2_9 = qdg_0[6] * e_dg_9; + float dk2_9 = kdg_0[6] * e_dg_9; + dbp_2[6] = dk2_9 * k_t_9; + float dk2b_9 = dk2_9 * bt_t_9; + float qe_8 = q_t_1_9 * e_dg_9; + float kp_20 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_20; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_20 * e_dg_9) * kdg_0[6]); + float dqf_9 = pool_f32[ew_2 + 768 + d_lane]; + float dkf_9 = pool_f32[ew_2 + 1792 + d_lane]; + float dgf_9 = pool_f32[ew_2 + 2816 + d_lane]; + float dq_o_9 = dq2_9 + dqf_9; + float dk_o_9 = dk2b_9 + dkf_9 + dkt_3[6]; + float dg_o_9 = __fmaf_rn(q_t_1_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_3[6], k_t_9, cr_8)) + dgf_9; + s_run_2[0] = s_run_2[0] + dg_o_9; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 6)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_25 = t_25 * qrs; - if (t_25 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_17 = t_17 * qrs; + if (t_17 < clen) { +#else + int qoff_9 = t_9 * qrs; + if (t_9 < clen) { +#endif { - auto _gst_489 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_489)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_489) + (0)) = (__nv_bfloat16)dq_o_25; +#if __CUDA_ARCH__ == 1030 + auto _gst_507 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_507)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_507) + (0)) = (__nv_bfloat16)dq_o_17; +#else + auto _gst_493 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_493)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_9; +#endif } { - auto _gst_490 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_490)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_490) + (0)) = (__nv_bfloat16)dk_o_25; +#if __CUDA_ARCH__ == 1030 + auto _gst_508 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_508)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_508) + (0)) = (__nv_bfloat16)dk_o_17; +#else + auto _gst_494 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_494)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_9; +#endif } } - int t_26 = 8 * th + 5; - float r_329[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_329[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_26 = r_329[0]; - unsigned int r_330[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_330[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_26 = __uint_as_float(r_330[0] << 16); - unsigned int r_331[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_26 = __uint_as_float(r_331[0] << 16); - float q_t_1_26 = ((t_26 >= clen) ? 0.0f : q_t_26); - float dqf_26 = pool_f32[ew_2 + 640 + d_lane]; - float dkf_26 = pool_f32[ew_2 + 1664 + d_lane]; - float dgf_26 = pool_f32[ew_2 + 2688 + d_lane]; - float bt_t_26 = btv_2[5]; - float e_off_26 = approx_exp2(g_t_26 - gn_2); - float e_dg_26 = approx_exp2(g_t_26 - gc_3); - float dq2_26 = qdg_2[5] * e_dg_26; - float dk2_26 = kdg_2[5] * e_dg_26; - dbp_2[5] = dk2_26 * k_t_26; - float dk2b_26 = dk2_26 * bt_t_26; - float dq_o_26 = dq2_26 + dqf_26; - float dk_o_26 = dk2b_26 + dkf_26 + dkt_3[5]; - float dg_o_26 = __fmaf_rn(q_t_1_26, dq2_26, (dk2b_26 - dkt_3[5]) * k_t_26) + dgf_26; - s_run_2[0] = s_run_2[0] + dg_o_26; +#if __CUDA_ARCH__ == 1030 + int t_18 = 8 * th + 5; + float r_308[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_308[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_18 = r_308[0]; + unsigned int r_309[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_18 = __uint_as_float(r_309[0] << 16); + unsigned int r_310[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_310[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_18 = __uint_as_float(r_310[0] << 16); + float q_t_1_18 = ((t_18 >= clen) ? 0.0f : q_t_18); + float bt_t_18 = btv_1_1[5]; + float e_off_18 = approx_exp2(g_t_18 - gn_2); + float e_dg_18 = approx_exp2(g_t_18 - gc_3); + float dq2_18 = qdg_1_1[5] * e_dg_18; + float dk2_18 = kdg_1_1[5] * e_dg_18; + dbp_2[5] = dk2_18 * k_t_18; + float dk2b_18 = dk2_18 * bt_t_18; + float qe_17 = q_t_1_18 * e_dg_18; + float kp_29 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_29; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1_1[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_29 * e_dg_18) * kdg_1_1[5]); + float dqf_18 = pool_f32[ew_2 + 640 + d_lane]; + float dkf_18 = pool_f32[ew_2 + 1664 + d_lane]; + float dgf_18 = pool_f32[ew_2 + 2688 + d_lane]; + float dq_o_18 = dq2_18 + dqf_18; + float dk_o_18 = dk2b_18 + dkf_18 + dkt_3[5]; + float dg_o_18 = __fmaf_rn(q_t_1_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_3[5], k_t_18, cr_17)) + dgf_18; + s_run_2[0] = s_run_2[0] + dg_o_18; +#else + int t_10 = 8 * th + 5; + float r_287[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 640 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_10 = r_287[0]; + unsigned int r_288[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_288[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_10 = __uint_as_float(r_288[0] << 16); + unsigned int r_289[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_289[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_10 = __uint_as_float(r_289[0] << 16); + float q_t_1_10 = ((t_10 >= clen) ? 0.0f : q_t_10); + float bt_t_10 = btv_0[5]; + float e_off_10 = approx_exp2(g_t_10 - gn_2); + float e_dg_10 = approx_exp2(g_t_10 - gc_3); + float dq2_10 = qdg_0[5] * e_dg_10; + float dk2_10 = kdg_0[5] * e_dg_10; + dbp_2[5] = dk2_10 * k_t_10; + float dk2b_10 = dk2_10 * bt_t_10; + float qe_9 = q_t_1_10 * e_dg_10; + float kp_21 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_21; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_21 * e_dg_10) * kdg_0[5]); + float dqf_10 = pool_f32[ew_2 + 640 + d_lane]; + float dkf_10 = pool_f32[ew_2 + 1664 + d_lane]; + float dgf_10 = pool_f32[ew_2 + 2688 + d_lane]; + float dq_o_10 = dq2_10 + dqf_10; + float dk_o_10 = dk2b_10 + dkf_10 + dkt_3[5]; + float dg_o_10 = __fmaf_rn(q_t_1_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_3[5], k_t_10, cr_9)) + dgf_10; + s_run_2[0] = s_run_2[0] + dg_o_10; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 5)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_26 = t_26 * qrs; - if (t_26 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_18 = t_18 * qrs; + if (t_18 < clen) { +#else + int qoff_10 = t_10 * qrs; + if (t_10 < clen) { +#endif { - auto _gst_491 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_491)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_491) + (0)) = (__nv_bfloat16)dq_o_26; +#if __CUDA_ARCH__ == 1030 + auto _gst_509 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_509)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_509) + (0)) = (__nv_bfloat16)dq_o_18; +#else + auto _gst_495 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_495)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_10; +#endif } { - auto _gst_492 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_492)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_492) + (0)) = (__nv_bfloat16)dk_o_26; +#if __CUDA_ARCH__ == 1030 + auto _gst_510 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_510)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_510) + (0)) = (__nv_bfloat16)dk_o_18; +#else + auto _gst_496 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_496)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_10; +#endif } } - int t_27 = 8 * th + 4; - float r_332[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_332[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_27 = r_332[0]; - unsigned int r_333[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_333[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_27 = __uint_as_float(r_333[0] << 16); - unsigned int r_334[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_27 = __uint_as_float(r_334[0] << 16); - float q_t_1_27 = ((t_27 >= clen) ? 0.0f : q_t_27); - float dqf_27 = pool_f32[ew_2 + 512 + d_lane]; - float dkf_27 = pool_f32[ew_2 + 1536 + d_lane]; - float dgf_27 = pool_f32[ew_2 + 2560 + d_lane]; - float bt_t_27 = btv_2[4]; - float e_off_27 = approx_exp2(g_t_27 - gn_2); - float e_dg_27 = approx_exp2(g_t_27 - gc_3); - float dq2_27 = qdg_2[4] * e_dg_27; - float dk2_27 = kdg_2[4] * e_dg_27; - dbp_2[4] = dk2_27 * k_t_27; - float dk2b_27 = dk2_27 * bt_t_27; - float dq_o_27 = dq2_27 + dqf_27; - float dk_o_27 = dk2b_27 + dkf_27 + dkt_3[4]; - float dg_o_27 = __fmaf_rn(q_t_1_27, dq2_27, (dk2b_27 - dkt_3[4]) * k_t_27) + dgf_27; - s_run_2[0] = s_run_2[0] + dg_o_27; +#if __CUDA_ARCH__ == 1030 + int t_19 = 8 * th + 4; + float r_311[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_311[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_19 = r_311[0]; + unsigned int r_312[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_19 = __uint_as_float(r_312[0] << 16); + unsigned int r_313[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_313[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_19 = __uint_as_float(r_313[0] << 16); + float q_t_1_19 = ((t_19 >= clen) ? 0.0f : q_t_19); + float bt_t_19 = btv_1_1[4]; + float e_off_19 = approx_exp2(g_t_19 - gn_2); + float e_dg_19 = approx_exp2(g_t_19 - gc_3); + float dq2_19 = qdg_1_1[4] * e_dg_19; + float dk2_19 = kdg_1_1[4] * e_dg_19; + dbp_2[4] = dk2_19 * k_t_19; + float dk2b_19 = dk2_19 * bt_t_19; + float qe_18 = q_t_1_19 * e_dg_19; + float kp_30 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_30; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1_1[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_30 * e_dg_19) * kdg_1_1[4]); + float dqf_19 = pool_f32[ew_2 + 512 + d_lane]; + float dkf_19 = pool_f32[ew_2 + 1536 + d_lane]; + float dgf_19 = pool_f32[ew_2 + 2560 + d_lane]; + float dq_o_19 = dq2_19 + dqf_19; + float dk_o_19 = dk2b_19 + dkf_19 + dkt_3[4]; + float dg_o_19 = __fmaf_rn(q_t_1_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_3[4], k_t_19, cr_18)) + dgf_19; + s_run_2[0] = s_run_2[0] + dg_o_19; +#else + int t_11 = 8 * th + 4; + float r_290[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 512 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_11 = r_290[0]; + unsigned int r_291[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_291[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_11 = __uint_as_float(r_291[0] << 16); + unsigned int r_292[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_292[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_11 = __uint_as_float(r_292[0] << 16); + float q_t_1_11 = ((t_11 >= clen) ? 0.0f : q_t_11); + float bt_t_11 = btv_0[4]; + float e_off_11 = approx_exp2(g_t_11 - gn_2); + float e_dg_11 = approx_exp2(g_t_11 - gc_3); + float dq2_11 = qdg_0[4] * e_dg_11; + float dk2_11 = kdg_0[4] * e_dg_11; + dbp_2[4] = dk2_11 * k_t_11; + float dk2b_11 = dk2_11 * bt_t_11; + float qe_10 = q_t_1_11 * e_dg_11; + float kp_22 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_22; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_22 * e_dg_11) * kdg_0[4]); + float dqf_11 = pool_f32[ew_2 + 512 + d_lane]; + float dkf_11 = pool_f32[ew_2 + 1536 + d_lane]; + float dgf_11 = pool_f32[ew_2 + 2560 + d_lane]; + float dq_o_11 = dq2_11 + dqf_11; + float dk_o_11 = dk2b_11 + dkf_11 + dkt_3[4]; + float dg_o_11 = __fmaf_rn(q_t_1_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_3[4], k_t_11, cr_10)) + dgf_11; + s_run_2[0] = s_run_2[0] + dg_o_11; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 4)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_27 = t_27 * qrs; - if (t_27 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_19 = t_19 * qrs; + if (t_19 < clen) { +#else + int qoff_11 = t_11 * qrs; + if (t_11 < clen) { +#endif { - auto _gst_493 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_493)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_493) + (0)) = (__nv_bfloat16)dq_o_27; +#if __CUDA_ARCH__ == 1030 + auto _gst_511 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_511)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_511) + (0)) = (__nv_bfloat16)dq_o_19; +#else + auto _gst_497 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_497)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_11; +#endif } { - auto _gst_494 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_494)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_494) + (0)) = (__nv_bfloat16)dk_o_27; +#if __CUDA_ARCH__ == 1030 + auto _gst_512 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_512)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_512) + (0)) = (__nv_bfloat16)dk_o_19; +#else + auto _gst_498 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_498)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_11; +#endif } } - int t_28 = 8 * th + 3; - float r_335[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_335[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_28 = r_335[0]; - unsigned int r_336[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_336[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_28 = __uint_as_float(r_336[0] << 16); - unsigned int r_337[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_28 = __uint_as_float(r_337[0] << 16); - float q_t_1_28 = ((t_28 >= clen) ? 0.0f : q_t_28); - float dqf_28 = pool_f32[ew_2 + 384 + d_lane]; - float dkf_28 = pool_f32[ew_2 + 1408 + d_lane]; - float dgf_28 = pool_f32[ew_2 + 2432 + d_lane]; - float bt_t_28 = btv_2[3]; - float e_off_28 = approx_exp2(g_t_28 - gn_2); - float e_dg_28 = approx_exp2(g_t_28 - gc_3); - float dq2_28 = qdg_2[3] * e_dg_28; - float dk2_28 = kdg_2[3] * e_dg_28; - dbp_2[3] = dk2_28 * k_t_28; - float dk2b_28 = dk2_28 * bt_t_28; - float dq_o_28 = dq2_28 + dqf_28; - float dk_o_28 = dk2b_28 + dkf_28 + dkt_3[3]; - float dg_o_28 = __fmaf_rn(q_t_1_28, dq2_28, (dk2b_28 - dkt_3[3]) * k_t_28) + dgf_28; - s_run_2[0] = s_run_2[0] + dg_o_28; +#if __CUDA_ARCH__ == 1030 + int t_20 = 8 * th + 3; + float r_314[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_314[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_20 = r_314[0]; + unsigned int r_315[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_20 = __uint_as_float(r_315[0] << 16); + unsigned int r_316[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_316[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_20 = __uint_as_float(r_316[0] << 16); + float q_t_1_20 = ((t_20 >= clen) ? 0.0f : q_t_20); + float bt_t_20 = btv_1_1[3]; + float e_off_20 = approx_exp2(g_t_20 - gn_2); + float e_dg_20 = approx_exp2(g_t_20 - gc_3); + float dq2_20 = qdg_1_1[3] * e_dg_20; + float dk2_20 = kdg_1_1[3] * e_dg_20; + dbp_2[3] = dk2_20 * k_t_20; + float dk2b_20 = dk2_20 * bt_t_20; + float qe_19 = q_t_1_20 * e_dg_20; + float kp_31 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_31; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1_1[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_31 * e_dg_20) * kdg_1_1[3]); + float dqf_20 = pool_f32[ew_2 + 384 + d_lane]; + float dkf_20 = pool_f32[ew_2 + 1408 + d_lane]; + float dgf_20 = pool_f32[ew_2 + 2432 + d_lane]; + float dq_o_20 = dq2_20 + dqf_20; + float dk_o_20 = dk2b_20 + dkf_20 + dkt_3[3]; + float dg_o_20 = __fmaf_rn(q_t_1_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_3[3], k_t_20, cr_19)) + dgf_20; + s_run_2[0] = s_run_2[0] + dg_o_20; +#else + int t_12 = 8 * th + 3; + float r_293[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 384 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_12 = r_293[0]; + unsigned int r_294[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_294[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_12 = __uint_as_float(r_294[0] << 16); + unsigned int r_295[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_295[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_12 = __uint_as_float(r_295[0] << 16); + float q_t_1_12 = ((t_12 >= clen) ? 0.0f : q_t_12); + float bt_t_12 = btv_0[3]; + float e_off_12 = approx_exp2(g_t_12 - gn_2); + float e_dg_12 = approx_exp2(g_t_12 - gc_3); + float dq2_12 = qdg_0[3] * e_dg_12; + float dk2_12 = kdg_0[3] * e_dg_12; + dbp_2[3] = dk2_12 * k_t_12; + float dk2b_12 = dk2_12 * bt_t_12; + float qe_11 = q_t_1_12 * e_dg_12; + float kp_23 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_23; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_23 * e_dg_12) * kdg_0[3]); + float dqf_12 = pool_f32[ew_2 + 384 + d_lane]; + float dkf_12 = pool_f32[ew_2 + 1408 + d_lane]; + float dgf_12 = pool_f32[ew_2 + 2432 + d_lane]; + float dq_o_12 = dq2_12 + dqf_12; + float dk_o_12 = dk2b_12 + dkf_12 + dkt_3[3]; + float dg_o_12 = __fmaf_rn(q_t_1_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_3[3], k_t_12, cr_11)) + dgf_12; + s_run_2[0] = s_run_2[0] + dg_o_12; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 3)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_28 = t_28 * qrs; - if (t_28 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_20 = t_20 * qrs; + if (t_20 < clen) { +#else + int qoff_12 = t_12 * qrs; + if (t_12 < clen) { +#endif { - auto _gst_495 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_495)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_495) + (0)) = (__nv_bfloat16)dq_o_28; +#if __CUDA_ARCH__ == 1030 + auto _gst_513 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_513)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_513) + (0)) = (__nv_bfloat16)dq_o_20; +#else + auto _gst_499 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_499)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_12; +#endif } { - auto _gst_496 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_496)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_496) + (0)) = (__nv_bfloat16)dk_o_28; +#if __CUDA_ARCH__ == 1030 + auto _gst_514 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_514)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_514) + (0)) = (__nv_bfloat16)dk_o_20; +#else + auto _gst_500 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_500)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_12; +#endif } } - int t_29 = 8 * th + 2; - float r_338[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_338[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_29 = r_338[0]; - unsigned int r_339[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_339[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_29 = __uint_as_float(r_339[0] << 16); - unsigned int r_340[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_29 = __uint_as_float(r_340[0] << 16); - float q_t_1_29 = ((t_29 >= clen) ? 0.0f : q_t_29); - float dqf_29 = pool_f32[ew_2 + 256 + d_lane]; - float dkf_29 = pool_f32[ew_2 + 1280 + d_lane]; - float dgf_29 = pool_f32[ew_2 + 2304 + d_lane]; - float bt_t_29 = btv_2[2]; - float e_off_29 = approx_exp2(g_t_29 - gn_2); - float e_dg_29 = approx_exp2(g_t_29 - gc_3); - float dq2_29 = qdg_2[2] * e_dg_29; - float dk2_29 = kdg_2[2] * e_dg_29; - dbp_2[2] = dk2_29 * k_t_29; - float dk2b_29 = dk2_29 * bt_t_29; - float dq_o_29 = dq2_29 + dqf_29; - float dk_o_29 = dk2b_29 + dkf_29 + dkt_3[2]; - float dg_o_29 = __fmaf_rn(q_t_1_29, dq2_29, (dk2b_29 - dkt_3[2]) * k_t_29) + dgf_29; - s_run_2[0] = s_run_2[0] + dg_o_29; +#if __CUDA_ARCH__ == 1030 + int t_21 = 8 * th + 2; + float r_317[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_317[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_21 = r_317[0]; + unsigned int r_318[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_21 = __uint_as_float(r_318[0] << 16); + unsigned int r_319[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_319[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_21 = __uint_as_float(r_319[0] << 16); + float q_t_1_21 = ((t_21 >= clen) ? 0.0f : q_t_21); + float bt_t_21 = btv_1_1[2]; + float e_off_21 = approx_exp2(g_t_21 - gn_2); + float e_dg_21 = approx_exp2(g_t_21 - gc_3); + float dq2_21 = qdg_1_1[2] * e_dg_21; + float dk2_21 = kdg_1_1[2] * e_dg_21; + dbp_2[2] = dk2_21 * k_t_21; + float dk2b_21 = dk2_21 * bt_t_21; + float qe_20 = q_t_1_21 * e_dg_21; + float kp_32 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_32; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1_1[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_32 * e_dg_21) * kdg_1_1[2]); + float dqf_21 = pool_f32[ew_2 + 256 + d_lane]; + float dkf_21 = pool_f32[ew_2 + 1280 + d_lane]; + float dgf_21 = pool_f32[ew_2 + 2304 + d_lane]; + float dq_o_21 = dq2_21 + dqf_21; + float dk_o_21 = dk2b_21 + dkf_21 + dkt_3[2]; + float dg_o_21 = __fmaf_rn(q_t_1_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_3[2], k_t_21, cr_20)) + dgf_21; + s_run_2[0] = s_run_2[0] + dg_o_21; +#else + int t_13 = 8 * th + 2; + float r_296[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 256 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_13 = r_296[0]; + unsigned int r_297[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_297[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_13 = __uint_as_float(r_297[0] << 16); + unsigned int r_298[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_298[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_13 = __uint_as_float(r_298[0] << 16); + float q_t_1_13 = ((t_13 >= clen) ? 0.0f : q_t_13); + float bt_t_13 = btv_0[2]; + float e_off_13 = approx_exp2(g_t_13 - gn_2); + float e_dg_13 = approx_exp2(g_t_13 - gc_3); + float dq2_13 = qdg_0[2] * e_dg_13; + float dk2_13 = kdg_0[2] * e_dg_13; + dbp_2[2] = dk2_13 * k_t_13; + float dk2b_13 = dk2_13 * bt_t_13; + float qe_12 = q_t_1_13 * e_dg_13; + float kp_24 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_24; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_24 * e_dg_13) * kdg_0[2]); + float dqf_13 = pool_f32[ew_2 + 256 + d_lane]; + float dkf_13 = pool_f32[ew_2 + 1280 + d_lane]; + float dgf_13 = pool_f32[ew_2 + 2304 + d_lane]; + float dq_o_13 = dq2_13 + dqf_13; + float dk_o_13 = dk2b_13 + dkf_13 + dkt_3[2]; + float dg_o_13 = __fmaf_rn(q_t_1_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_3[2], k_t_13, cr_12)) + dgf_13; + s_run_2[0] = s_run_2[0] + dg_o_13; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 2)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_29 = t_29 * qrs; - if (t_29 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_21 = t_21 * qrs; + if (t_21 < clen) { +#else + int qoff_13 = t_13 * qrs; + if (t_13 < clen) { +#endif { - auto _gst_497 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_497)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_497) + (0)) = (__nv_bfloat16)dq_o_29; +#if __CUDA_ARCH__ == 1030 + auto _gst_515 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_515)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_515) + (0)) = (__nv_bfloat16)dq_o_21; +#else + auto _gst_501 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_501)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_13; +#endif } { - auto _gst_498 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_498)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_498) + (0)) = (__nv_bfloat16)dk_o_29; +#if __CUDA_ARCH__ == 1030 + auto _gst_516 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_516)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_516) + (0)) = (__nv_bfloat16)dk_o_21; +#else + auto _gst_502 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_502)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_13; +#endif } } - int t_30 = 8 * th + 1; - float r_341[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_341[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_30 = r_341[0]; - unsigned int r_342[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_342[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_30 = __uint_as_float(r_342[0] << 16); - unsigned int r_343[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_30 = __uint_as_float(r_343[0] << 16); - float q_t_1_30 = ((t_30 >= clen) ? 0.0f : q_t_30); - float dqf_30 = pool_f32[ew_2 + 128 + d_lane]; - float dkf_30 = pool_f32[ew_2 + 1152 + d_lane]; - float dgf_30 = pool_f32[ew_2 + 2176 + d_lane]; - float bt_t_30 = btv_2[1]; - float e_off_30 = approx_exp2(g_t_30 - gn_2); - float e_dg_30 = approx_exp2(g_t_30 - gc_3); - float dq2_30 = qdg_2[1] * e_dg_30; - float dk2_30 = kdg_2[1] * e_dg_30; - dbp_2[1] = dk2_30 * k_t_30; - float dk2b_30 = dk2_30 * bt_t_30; - float dq_o_30 = dq2_30 + dqf_30; - float dk_o_30 = dk2b_30 + dkf_30 + dkt_3[1]; - float dg_o_30 = __fmaf_rn(q_t_1_30, dq2_30, (dk2b_30 - dkt_3[1]) * k_t_30) + dgf_30; - s_run_2[0] = s_run_2[0] + dg_o_30; +#if __CUDA_ARCH__ == 1030 + int t_22 = 8 * th + 1; + float r_320[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_320[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_22 = r_320[0]; + unsigned int r_321[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_22 = __uint_as_float(r_321[0] << 16); + unsigned int r_322[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_322[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_22 = __uint_as_float(r_322[0] << 16); + float q_t_1_22 = ((t_22 >= clen) ? 0.0f : q_t_22); + float bt_t_22 = btv_1_1[1]; + float e_off_22 = approx_exp2(g_t_22 - gn_2); + float e_dg_22 = approx_exp2(g_t_22 - gc_3); + float dq2_22 = qdg_1_1[1] * e_dg_22; + float dk2_22 = kdg_1_1[1] * e_dg_22; + dbp_2[1] = dk2_22 * k_t_22; + float dk2b_22 = dk2_22 * bt_t_22; + float qe_21 = q_t_1_22 * e_dg_22; + float kp_33 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_33; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1_1[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_33 * e_dg_22) * kdg_1_1[1]); + float dqf_22 = pool_f32[ew_2 + 128 + d_lane]; + float dkf_22 = pool_f32[ew_2 + 1152 + d_lane]; + float dgf_22 = pool_f32[ew_2 + 2176 + d_lane]; + float dq_o_22 = dq2_22 + dqf_22; + float dk_o_22 = dk2b_22 + dkf_22 + dkt_3[1]; + float dg_o_22 = __fmaf_rn(q_t_1_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_3[1], k_t_22, cr_21)) + dgf_22; + s_run_2[0] = s_run_2[0] + dg_o_22; +#else + int t_14 = 8 * th + 1; + float r_299[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_299[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + 128 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_14 = r_299[0]; + unsigned int r_300[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_14 = __uint_as_float(r_300[0] << 16); + unsigned int r_301[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_301[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_14 = __uint_as_float(r_301[0] << 16); + float q_t_1_14 = ((t_14 >= clen) ? 0.0f : q_t_14); + float bt_t_14 = btv_0[1]; + float e_off_14 = approx_exp2(g_t_14 - gn_2); + float e_dg_14 = approx_exp2(g_t_14 - gc_3); + float dq2_14 = qdg_0[1] * e_dg_14; + float dk2_14 = kdg_0[1] * e_dg_14; + dbp_2[1] = dk2_14 * k_t_14; + float dk2b_14 = dk2_14 * bt_t_14; + float qe_13 = q_t_1_14 * e_dg_14; + float kp_25 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_25; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_25 * e_dg_14) * kdg_0[1]); + float dqf_14 = pool_f32[ew_2 + 128 + d_lane]; + float dkf_14 = pool_f32[ew_2 + 1152 + d_lane]; + float dgf_14 = pool_f32[ew_2 + 2176 + d_lane]; + float dq_o_14 = dq2_14 + dqf_14; + float dk_o_14 = dk2b_14 + dkf_14 + dkt_3[1]; + float dg_o_14 = __fmaf_rn(q_t_1_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_3[1], k_t_14, cr_13)) + dgf_14; + s_run_2[0] = s_run_2[0] + dg_o_14; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th + 1)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_30 = t_30 * qrs; - if (t_30 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_22 = t_22 * qrs; + if (t_22 < clen) { +#else + int qoff_14 = t_14 * qrs; + if (t_14 < clen) { +#endif { - auto _gst_499 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_499)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_499) + (0)) = (__nv_bfloat16)dq_o_30; +#if __CUDA_ARCH__ == 1030 + auto _gst_517 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_517)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_517) + (0)) = (__nv_bfloat16)dq_o_22; +#else + auto _gst_503 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_503)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_503) + (0)) = (__nv_bfloat16)dq_o_14; +#endif } { - auto _gst_500 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_500)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_500) + (0)) = (__nv_bfloat16)dk_o_30; +#if __CUDA_ARCH__ == 1030 + auto _gst_518 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_518)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_518) + (0)) = (__nv_bfloat16)dk_o_22; +#else + auto _gst_504 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_504)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_504) + (0)) = (__nv_bfloat16)dk_o_14; +#endif } } - int t_31 = 0 + 8 * th; - float r_344[1]; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_344[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); - float g_t_31 = r_344[0]; - unsigned int r_345[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_345[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float k_t_31 = __uint_as_float(r_345[0] << 16); - unsigned int r_346[1]; - asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); - float q_t_31 = __uint_as_float(r_346[0] << 16); - float q_t_1_31 = ((t_31 >= clen) ? 0.0f : q_t_31); - float dqf_31 = pool_f32[ew_2 + d_lane]; - float dkf_31 = pool_f32[ew_2 + 1024 + d_lane]; - float dgf_31 = pool_f32[ew_2 + 2048 + d_lane]; - float bt_t_31 = btv_2[0]; - float e_off_31 = approx_exp2(g_t_31 - gn_2); - float e_dg_31 = approx_exp2(g_t_31 - gc_3); - float dq2_31 = qdg_2[0] * e_dg_31; - float dk2_31 = kdg_2[0] * e_dg_31; - dbp_2[0] = dk2_31 * k_t_31; - float dk2b_31 = dk2_31 * bt_t_31; - float dq_o_31 = dq2_31 + dqf_31; - float dk_o_31 = dk2b_31 + dkf_31 + dkt_3[0]; - float dg_o_31 = __fmaf_rn(q_t_1_31, dq2_31, (dk2b_31 - dkt_3[0]) * k_t_31) + dgf_31; - s_run_2[0] = s_run_2[0] + dg_o_31; +#if __CUDA_ARCH__ == 1030 + int t_23 = 0 + 8 * th; + float r_323[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_323[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_23 = r_323[0]; + unsigned int r_324[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_324[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_23 = __uint_as_float(r_324[0] << 16); + unsigned int r_325[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_23 = __uint_as_float(r_325[0] << 16); + float q_t_1_23 = ((t_23 >= clen) ? 0.0f : q_t_23); + float bt_t_23 = btv_1_1[0]; + float e_off_23 = approx_exp2(g_t_23 - gn_2); + float e_dg_23 = approx_exp2(g_t_23 - gc_3); + float dq2_23 = qdg_1_1[0] * e_dg_23; + float dk2_23 = kdg_1_1[0] * e_dg_23; + dbp_2[0] = dk2_23 * k_t_23; + float dk2b_23 = dk2_23 * bt_t_23; + float qe_22 = q_t_1_23 * e_dg_23; + float kp_34 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_34; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1_1[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_34 * e_dg_23) * kdg_1_1[0]); + float dqf_23 = pool_f32[ew_2 + d_lane]; + float dkf_23 = pool_f32[ew_2 + 1024 + d_lane]; + float dgf_23 = pool_f32[ew_2 + 2048 + d_lane]; + float dq_o_23 = dq2_23 + dqf_23; + float dk_o_23 = dk2b_23 + dkf_23 + dkt_3[0]; + float dg_o_23 = __fmaf_rn(q_t_1_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_3[0], k_t_23, cr_22)) + dgf_23; + s_run_2[0] = s_run_2[0] + dg_o_23; +#else + int t_15 = 0 + 8 * th; + float r_302[1]; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&r_302[0])) : "r"((gb8_2 + (lane_1 / 32 * 8192 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 8192 + lane_1 % 32 * 4 >> 7 & 7) << 4)))); + float g_t_15 = r_302[0]; + unsigned int r_303[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((kb8_2 + (dcol / 64 * 8192 + dcol % 64 * 2 ^ (dcol / 64 * 8192 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float k_t_15 = __uint_as_float(r_303[0] << 16); + unsigned int r_304[1]; + asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_304[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); + float q_t_15 = __uint_as_float(r_304[0] << 16); + float q_t_1_15 = ((t_15 >= clen) ? 0.0f : q_t_15); + float bt_t_15 = btv_0[0]; + float e_off_15 = approx_exp2(g_t_15 - gn_2); + float e_dg_15 = approx_exp2(g_t_15 - gc_3); + float dq2_15 = qdg_0[0] * e_dg_15; + float dk2_15 = kdg_0[0] * e_dg_15; + dbp_2[0] = dk2_15 * k_t_15; + float dk2b_15 = dk2_15 * bt_t_15; + float qe_14 = q_t_1_15 * e_dg_15; + float kp_26 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_26; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_26 * e_dg_15) * kdg_0[0]); + float dqf_15 = pool_f32[ew_2 + d_lane]; + float dkf_15 = pool_f32[ew_2 + 1024 + d_lane]; + float dgf_15 = pool_f32[ew_2 + 2048 + d_lane]; + float dq_o_15 = dq2_15 + dqf_15; + float dk_o_15 = dk2b_15 + dkf_15 + dkt_3[0]; + float dg_o_15 = __fmaf_rn(q_t_1_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_3[0], k_t_15, cr_14)) + dgf_15; + s_run_2[0] = s_run_2[0] + dg_o_15; +#endif asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" " [%0], {%1};" :: "r"(taddr + (unsigned int)row_base + (unsigned int)(128 + 8 * th)), "f"(s_run_2[0])); asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); - int qoff_31 = t_31 * qrs; - if (t_31 < clen) { +#if __CUDA_ARCH__ == 1030 + int qoff_23 = t_23 * qrs; + if (t_23 < clen) { +#else + int qoff_15 = t_15 * qrs; + if (t_15 < clen) { +#endif { - auto _gst_501 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_501)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_501) + (0)) = (__nv_bfloat16)dq_o_31; +#if __CUDA_ARCH__ == 1030 + auto _gst_519 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_519)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_519) + (0)) = (__nv_bfloat16)dq_o_23; +#else + auto _gst_505 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_505)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_505) + (0)) = (__nv_bfloat16)dq_o_15; +#endif } { - auto _gst_502 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_502)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_502) + (0)) = (__nv_bfloat16)dk_o_31; +#if __CUDA_ARCH__ == 1030 + auto _gst_520 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_520)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_520) + (0)) = (__nv_bfloat16)dk_o_23; +#else + auto _gst_506 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_506)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_506) + (0)) = (__nv_bfloat16)dk_o_15; +#endif } } float tot_2 = s_run_2[0]; @@ -5626,37 +7387,81 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_503 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_503; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_521 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_521; +#else + float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_507; +#endif float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_504 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_504; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_522 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_522; +#else + float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_508; +#endif float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_505 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_505; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_523 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_523; +#else + float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_509; +#endif float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_506 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_506; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_524 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_524; +#else + float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_510; +#endif float v2_18[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_507 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_18[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_507; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_525 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_18[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_525; +#else + float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_18[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_511; +#endif float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_508 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_18[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_508; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_526 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_18[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_526; +#else + float _shfl_xor_sync_512 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_18[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_512; +#endif float send_26 = ((b2_2 == 0) ? v2_18[1] : v2_18[0]); - float _shfl_xor_sync_509 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_18[0] : v2_18[1]) + _shfl_xor_sync_509; - float _shfl_xor_sync_510 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_510; - float _shfl_xor_sync_511 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_511; +#if __CUDA_ARCH__ == 1030 + float _shfl_xor_sync_527 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_18[0] : v2_18[1]) + _shfl_xor_sync_527; + float _shfl_xor_sync_528 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_528; + float _shfl_xor_sync_529 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_529; +#else + float _shfl_xor_sync_513 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_18[0] : v2_18[1]) + _shfl_xor_sync_513; + float _shfl_xor_sync_514 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_514; + float _shfl_xor_sync_515 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_515; +#endif tmem_ld_x8(&gacc[0], taddr + (unsigned int)row_base + (unsigned int)(160 + 8 * th)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_512 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_512), "f"(tot_2) : "memory"); +#if __CUDA_ARCH__ == 1030 + uint32_t _addr_530 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_530), "f"(tot_2) : "memory"); +#else + uint32_t _addr_516 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_516), "f"(tot_2) : "memory"); +#endif } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -5688,132 +7493,280 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d float dgr8_2[8]; float2 dyg2_12 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[0], sfx_2[1])); float2 rp_u_11 = add_f32x2(make_float2(raw_t0[0], raw_t0[1]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_513 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_531 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_517 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_513), "l"(*(const unsigned long long*)&rp_u_11)); - float _abs_516; - asm("abs.f32 %0, %1;" : "=f"(_abs_516) : "f"(rp_z_11.x)); - float _neg_515; - asm("neg.f32 %0, %1;" : "=f"(_neg_515) : "f"(_abs_516)); - float _exp_514 = __nv_expf(_neg_515); - float _abs_519; - asm("abs.f32 %0, %1;" : "=f"(_abs_519) : "f"(rp_z_11.y)); - float _neg_518; - asm("neg.f32 %0, %1;" : "=f"(_neg_518) : "f"(_abs_519)); - float _exp_517 = __nv_expf(_neg_518); - float2 rp_e_11 = make_float2(_exp_514, _exp_517); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_531), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_534; + asm("abs.f32 %0, %1;" : "=f"(_abs_534) : "f"(rp_z_11.x)); + float _neg_533; + asm("neg.f32 %0, %1;" : "=f"(_neg_533) : "f"(_abs_534)); + float _exp_532 = __nv_expf(_neg_533); + float _abs_537; + asm("abs.f32 %0, %1;" : "=f"(_abs_537) : "f"(rp_z_11.y)); + float _neg_536; + asm("neg.f32 %0, %1;" : "=f"(_neg_536) : "f"(_abs_537)); + float _exp_535 = __nv_expf(_neg_536); + float2 rp_e_11 = make_float2(_exp_532, _exp_535); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_517), "l"(*(const unsigned long long*)&rp_u_11)); + float _abs_520; + asm("abs.f32 %0, %1;" : "=f"(_abs_520) : "f"(rp_z_11.x)); + float _neg_519; + asm("neg.f32 %0, %1;" : "=f"(_neg_519) : "f"(_abs_520)); + float _exp_518 = __nv_expf(_neg_519); + float _abs_523; + asm("abs.f32 %0, %1;" : "=f"(_abs_523) : "f"(rp_z_11.y)); + float _neg_522; + asm("neg.f32 %0, %1;" : "=f"(_neg_522) : "f"(_abs_523)); + float _exp_521 = __nv_expf(_neg_522); + float2 rp_e_11 = make_float2(_exp_518, _exp_521); +#endif float2 rp_dn_11 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_11); float2 rp_den_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_11) : "l"(*(const unsigned long long*)&rp_dn_11), "l"(*(const unsigned long long*)&rp_dn_11)); - float _rcp_520 = approx_rcp(rp_den_11.x); - float _rcp_521 = approx_rcp(rp_den_11.y); - float2 _f32x2_mul_b_522 = make_float2(_rcp_520, _rcp_521); +#if __CUDA_ARCH__ == 1030 + float _rcp_538 = approx_rcp(rp_den_11.x); + float _rcp_539 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_540 = make_float2(_rcp_538, _rcp_539); +#else + float _rcp_524 = approx_rcp(rp_den_11.x); + float _rcp_525 = approx_rcp(rp_den_11.y); + float2 _f32x2_mul_b_526 = make_float2(_rcp_524, _rcp_525); +#endif float2 rp_sp_11; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_522)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_540)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_11) : "l"(*(const unsigned long long*)&rp_e_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_526)); +#endif float2 rp_t_11; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_11) : "l"(*(const unsigned long long*)&dyg2_12), "l"(*(const unsigned long long*)&rp_sp_11)); - float2 _f32x2_mul_b_523 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_541 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_527 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_523)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_541)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_11) : "l"(*(const unsigned long long*)&rp_t_11), "l"(*(const unsigned long long*)&_f32x2_mul_b_527)); +#endif u8_2[0] = rp_u_11.x; u8_2[1] = rp_u_11.y; dgr8_2[0] = rp_dgr_11.x; dgr8_2[1] = rp_dgr_11.y; float2 dyg2_13 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[2], sfx_2[3])); float2 rp_u_12 = add_f32x2(make_float2(raw_t0[2], raw_t0[3]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_524 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_542 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_528 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_524), "l"(*(const unsigned long long*)&rp_u_12)); - float _abs_527; - asm("abs.f32 %0, %1;" : "=f"(_abs_527) : "f"(rp_z_12.x)); - float _neg_526; - asm("neg.f32 %0, %1;" : "=f"(_neg_526) : "f"(_abs_527)); - float _exp_525 = __nv_expf(_neg_526); - float _abs_530; - asm("abs.f32 %0, %1;" : "=f"(_abs_530) : "f"(rp_z_12.y)); - float _neg_529; - asm("neg.f32 %0, %1;" : "=f"(_neg_529) : "f"(_abs_530)); - float _exp_528 = __nv_expf(_neg_529); - float2 rp_e_12 = make_float2(_exp_525, _exp_528); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_542), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_545; + asm("abs.f32 %0, %1;" : "=f"(_abs_545) : "f"(rp_z_12.x)); + float _neg_544; + asm("neg.f32 %0, %1;" : "=f"(_neg_544) : "f"(_abs_545)); + float _exp_543 = __nv_expf(_neg_544); + float _abs_548; + asm("abs.f32 %0, %1;" : "=f"(_abs_548) : "f"(rp_z_12.y)); + float _neg_547; + asm("neg.f32 %0, %1;" : "=f"(_neg_547) : "f"(_abs_548)); + float _exp_546 = __nv_expf(_neg_547); + float2 rp_e_12 = make_float2(_exp_543, _exp_546); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_528), "l"(*(const unsigned long long*)&rp_u_12)); + float _abs_531; + asm("abs.f32 %0, %1;" : "=f"(_abs_531) : "f"(rp_z_12.x)); + float _neg_530; + asm("neg.f32 %0, %1;" : "=f"(_neg_530) : "f"(_abs_531)); + float _exp_529 = __nv_expf(_neg_530); + float _abs_534; + asm("abs.f32 %0, %1;" : "=f"(_abs_534) : "f"(rp_z_12.y)); + float _neg_533; + asm("neg.f32 %0, %1;" : "=f"(_neg_533) : "f"(_abs_534)); + float _exp_532 = __nv_expf(_neg_533); + float2 rp_e_12 = make_float2(_exp_529, _exp_532); +#endif float2 rp_dn_12 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_12); float2 rp_den_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_12) : "l"(*(const unsigned long long*)&rp_dn_12), "l"(*(const unsigned long long*)&rp_dn_12)); - float _rcp_531 = approx_rcp(rp_den_12.x); - float _rcp_532 = approx_rcp(rp_den_12.y); - float2 _f32x2_mul_b_533 = make_float2(_rcp_531, _rcp_532); +#if __CUDA_ARCH__ == 1030 + float _rcp_549 = approx_rcp(rp_den_12.x); + float _rcp_550 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_551 = make_float2(_rcp_549, _rcp_550); +#else + float _rcp_535 = approx_rcp(rp_den_12.x); + float _rcp_536 = approx_rcp(rp_den_12.y); + float2 _f32x2_mul_b_537 = make_float2(_rcp_535, _rcp_536); +#endif float2 rp_sp_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_533)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_551)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_12) : "l"(*(const unsigned long long*)&rp_e_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_537)); +#endif float2 rp_t_12; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_12) : "l"(*(const unsigned long long*)&dyg2_13), "l"(*(const unsigned long long*)&rp_sp_12)); - float2 _f32x2_mul_b_534 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_552 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_538 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_534)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_552)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_12) : "l"(*(const unsigned long long*)&rp_t_12), "l"(*(const unsigned long long*)&_f32x2_mul_b_538)); +#endif u8_2[2] = rp_u_12.x; u8_2[3] = rp_u_12.y; dgr8_2[2] = rp_dgr_12.x; dgr8_2[3] = rp_dgr_12.y; float2 dyg2_14 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[4], sfx_2[5])); float2 rp_u_13 = add_f32x2(make_float2(raw_t0[4], raw_t0[5]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_535 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_553 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_539 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_535), "l"(*(const unsigned long long*)&rp_u_13)); - float _abs_538; - asm("abs.f32 %0, %1;" : "=f"(_abs_538) : "f"(rp_z_13.x)); - float _neg_537; - asm("neg.f32 %0, %1;" : "=f"(_neg_537) : "f"(_abs_538)); - float _exp_536 = __nv_expf(_neg_537); - float _abs_541; - asm("abs.f32 %0, %1;" : "=f"(_abs_541) : "f"(rp_z_13.y)); - float _neg_540; - asm("neg.f32 %0, %1;" : "=f"(_neg_540) : "f"(_abs_541)); - float _exp_539 = __nv_expf(_neg_540); - float2 rp_e_13 = make_float2(_exp_536, _exp_539); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_553), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_556; + asm("abs.f32 %0, %1;" : "=f"(_abs_556) : "f"(rp_z_13.x)); + float _neg_555; + asm("neg.f32 %0, %1;" : "=f"(_neg_555) : "f"(_abs_556)); + float _exp_554 = __nv_expf(_neg_555); + float _abs_559; + asm("abs.f32 %0, %1;" : "=f"(_abs_559) : "f"(rp_z_13.y)); + float _neg_558; + asm("neg.f32 %0, %1;" : "=f"(_neg_558) : "f"(_abs_559)); + float _exp_557 = __nv_expf(_neg_558); + float2 rp_e_13 = make_float2(_exp_554, _exp_557); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_539), "l"(*(const unsigned long long*)&rp_u_13)); + float _abs_542; + asm("abs.f32 %0, %1;" : "=f"(_abs_542) : "f"(rp_z_13.x)); + float _neg_541; + asm("neg.f32 %0, %1;" : "=f"(_neg_541) : "f"(_abs_542)); + float _exp_540 = __nv_expf(_neg_541); + float _abs_545; + asm("abs.f32 %0, %1;" : "=f"(_abs_545) : "f"(rp_z_13.y)); + float _neg_544; + asm("neg.f32 %0, %1;" : "=f"(_neg_544) : "f"(_abs_545)); + float _exp_543 = __nv_expf(_neg_544); + float2 rp_e_13 = make_float2(_exp_540, _exp_543); +#endif float2 rp_dn_13 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_13); float2 rp_den_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_13) : "l"(*(const unsigned long long*)&rp_dn_13), "l"(*(const unsigned long long*)&rp_dn_13)); - float _rcp_542 = approx_rcp(rp_den_13.x); - float _rcp_543 = approx_rcp(rp_den_13.y); - float2 _f32x2_mul_b_544 = make_float2(_rcp_542, _rcp_543); +#if __CUDA_ARCH__ == 1030 + float _rcp_560 = approx_rcp(rp_den_13.x); + float _rcp_561 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_562 = make_float2(_rcp_560, _rcp_561); +#else + float _rcp_546 = approx_rcp(rp_den_13.x); + float _rcp_547 = approx_rcp(rp_den_13.y); + float2 _f32x2_mul_b_548 = make_float2(_rcp_546, _rcp_547); +#endif float2 rp_sp_13; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_544)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_562)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_13) : "l"(*(const unsigned long long*)&rp_e_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_548)); +#endif float2 rp_t_13; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_13) : "l"(*(const unsigned long long*)&dyg2_14), "l"(*(const unsigned long long*)&rp_sp_13)); - float2 _f32x2_mul_b_545 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_563 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_549 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_545)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_563)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_13) : "l"(*(const unsigned long long*)&rp_t_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_549)); +#endif u8_2[4] = rp_u_13.x; u8_2[5] = rp_u_13.y; dgr8_2[4] = rp_dgr_13.x; dgr8_2[5] = rp_dgr_13.y; float2 dyg2_15 = add_f32x2(make_float2(offset_2, offset_2), make_float2(sfx_2[6], sfx_2[7])); float2 rp_u_14 = add_f32x2(make_float2(raw_t0[6], raw_t0[7]), make_float2(gacc[6], gacc[6])); - float2 _f32x2_mul_a_546 = make_float2(gacc[4], gacc[4]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_a_564 = make_float2(gacc[4], gacc[4]); +#else + float2 _f32x2_mul_a_550 = make_float2(gacc[4], gacc[4]); +#endif float2 rp_z_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_546), "l"(*(const unsigned long long*)&rp_u_14)); - float _abs_549; - asm("abs.f32 %0, %1;" : "=f"(_abs_549) : "f"(rp_z_14.x)); - float _neg_548; - asm("neg.f32 %0, %1;" : "=f"(_neg_548) : "f"(_abs_549)); - float _exp_547 = __nv_expf(_neg_548); - float _abs_552; - asm("abs.f32 %0, %1;" : "=f"(_abs_552) : "f"(rp_z_14.y)); - float _neg_551; - asm("neg.f32 %0, %1;" : "=f"(_neg_551) : "f"(_abs_552)); - float _exp_550 = __nv_expf(_neg_551); - float2 rp_e_14 = make_float2(_exp_547, _exp_550); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_564), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_567; + asm("abs.f32 %0, %1;" : "=f"(_abs_567) : "f"(rp_z_14.x)); + float _neg_566; + asm("neg.f32 %0, %1;" : "=f"(_neg_566) : "f"(_abs_567)); + float _exp_565 = __nv_expf(_neg_566); + float _abs_570; + asm("abs.f32 %0, %1;" : "=f"(_abs_570) : "f"(rp_z_14.y)); + float _neg_569; + asm("neg.f32 %0, %1;" : "=f"(_neg_569) : "f"(_abs_570)); + float _exp_568 = __nv_expf(_neg_569); + float2 rp_e_14 = make_float2(_exp_565, _exp_568); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_z_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_550), "l"(*(const unsigned long long*)&rp_u_14)); + float _abs_553; + asm("abs.f32 %0, %1;" : "=f"(_abs_553) : "f"(rp_z_14.x)); + float _neg_552; + asm("neg.f32 %0, %1;" : "=f"(_neg_552) : "f"(_abs_553)); + float _exp_551 = __nv_expf(_neg_552); + float _abs_556; + asm("abs.f32 %0, %1;" : "=f"(_abs_556) : "f"(rp_z_14.y)); + float _neg_555; + asm("neg.f32 %0, %1;" : "=f"(_neg_555) : "f"(_abs_556)); + float _exp_554 = __nv_expf(_neg_555); + float2 rp_e_14 = make_float2(_exp_551, _exp_554); +#endif float2 rp_dn_14 = add_f32x2(make_float2(1.0f, 1.0f), rp_e_14); float2 rp_den_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_den_14) : "l"(*(const unsigned long long*)&rp_dn_14), "l"(*(const unsigned long long*)&rp_dn_14)); - float _rcp_553 = approx_rcp(rp_den_14.x); - float _rcp_554 = approx_rcp(rp_den_14.y); - float2 _f32x2_mul_b_555 = make_float2(_rcp_553, _rcp_554); +#if __CUDA_ARCH__ == 1030 + float _rcp_571 = approx_rcp(rp_den_14.x); + float _rcp_572 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_573 = make_float2(_rcp_571, _rcp_572); +#else + float _rcp_557 = approx_rcp(rp_den_14.x); + float _rcp_558 = approx_rcp(rp_den_14.y); + float2 _f32x2_mul_b_559 = make_float2(_rcp_557, _rcp_558); +#endif float2 rp_sp_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_555)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_573)); +#else + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_sp_14) : "l"(*(const unsigned long long*)&rp_e_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_559)); +#endif float2 rp_t_14; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_t_14) : "l"(*(const unsigned long long*)&dyg2_15), "l"(*(const unsigned long long*)&rp_sp_14)); - float2 _f32x2_mul_b_556 = make_float2(gacc[5], gacc[5]); +#if __CUDA_ARCH__ == 1030 + float2 _f32x2_mul_b_574 = make_float2(gacc[5], gacc[5]); +#else + float2 _f32x2_mul_b_560 = make_float2(gacc[5], gacc[5]); +#endif float2 rp_dgr_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_556)); +#if __CUDA_ARCH__ == 1030 + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_574)); +#else + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&rp_dgr_14) : "l"(*(const unsigned long long*)&rp_t_14), "l"(*(const unsigned long long*)&_f32x2_mul_b_560)); +#endif u8_2[6] = rp_u_14.x; u8_2[7] = rp_u_14.y; dgr8_2[6] = rp_dgr_14.x; @@ -5828,54 +7781,102 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d } unsigned int pv_12 = __shfl_xor_sync(0xFFFFFFFF, pw_2[0], 1); unsigned int pq_12[1]; - uint32_t _prmt_557; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_557) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); - pq_12[0] = _prmt_557; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_575; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_575) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_575; +#else + uint32_t _prmt_561; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[0]), "r"(pv_12), "r"(sel_2)); + pq_12[0] = _prmt_561; +#endif int t_q_12 = 8 * th + odd_2; if (t_q_12 < clen) { { - auto _gst_558 = draw_base + (t_q_12 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_558)); - *(reinterpret_cast(_gst_558) + (0)) = pq_12[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_576 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_576)); + *(reinterpret_cast(_gst_576) + (0)) = pq_12[0]; +#else + auto _gst_562 = draw_base + (t_q_12 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_562)); + *(reinterpret_cast(_gst_562) + (0)) = pq_12[0]; +#endif } } unsigned int pv_13 = __shfl_xor_sync(0xFFFFFFFF, pw_2[1], 1); unsigned int pq_13[1]; - uint32_t _prmt_559; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_559) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); - pq_13[0] = _prmt_559; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_577; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_577) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_577; +#else + uint32_t _prmt_563; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[1]), "r"(pv_13), "r"(sel_2)); + pq_13[0] = _prmt_563; +#endif int t_q_13 = 8 * th + 2 + odd_2; if (t_q_13 < clen) { { - auto _gst_560 = draw_base + (t_q_13 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_560)); - *(reinterpret_cast(_gst_560) + (0)) = pq_13[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_578 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_578)); + *(reinterpret_cast(_gst_578) + (0)) = pq_13[0]; +#else + auto _gst_564 = draw_base + (t_q_13 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_564)); + *(reinterpret_cast(_gst_564) + (0)) = pq_13[0]; +#endif } } unsigned int pv_14 = __shfl_xor_sync(0xFFFFFFFF, pw_2[2], 1); unsigned int pq_14[1]; - uint32_t _prmt_561; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_561) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); - pq_14[0] = _prmt_561; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_579; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_579) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_579; +#else + uint32_t _prmt_565; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_565) : "r"(pw_2[2]), "r"(pv_14), "r"(sel_2)); + pq_14[0] = _prmt_565; +#endif int t_q_14 = 8 * th + 4 + odd_2; if (t_q_14 < clen) { { - auto _gst_562 = draw_base + (t_q_14 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_562)); - *(reinterpret_cast(_gst_562) + (0)) = pq_14[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_580 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_580)); + *(reinterpret_cast(_gst_580) + (0)) = pq_14[0]; +#else + auto _gst_566 = draw_base + (t_q_14 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_566)); + *(reinterpret_cast(_gst_566) + (0)) = pq_14[0]; +#endif } } unsigned int pv_15 = __shfl_xor_sync(0xFFFFFFFF, pw_2[3], 1); unsigned int pq_15[1]; - uint32_t _prmt_563; - asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_563) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); - pq_15[0] = _prmt_563; +#if __CUDA_ARCH__ == 1030 + uint32_t _prmt_581; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_581) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_581; +#else + uint32_t _prmt_567; + asm("prmt.b32 %0, %1, %2, %3;" : "=r"(_prmt_567) : "r"(pw_2[3]), "r"(pv_15), "r"(sel_2)); + pq_15[0] = _prmt_567; +#endif int t_q_15 = 8 * th + 6 + odd_2; if (t_q_15 < clen) { { - auto _gst_564 = draw_base + (t_q_15 * rs_g - odd_2); - __builtin_assume(__isGlobal(_gst_564)); - *(reinterpret_cast(_gst_564) + (0)) = pq_15[0]; +#if __CUDA_ARCH__ == 1030 + auto _gst_582 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_582)); + *(reinterpret_cast(_gst_582) + (0)) = pq_15[0]; +#else + auto _gst_568 = draw_base + (t_q_15 * rs_g - odd_2); + __builtin_assume(__isGlobal(_gst_568)); + *(reinterpret_cast(_gst_568) + (0)) = pq_15[0]; +#endif } } int t_a_24 = 8 * th; @@ -5940,47 +7941,99 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(pa_2) : "f"(pa01_2), "f"(pa23_2)); float nm_t_5; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_5) : "f"(s_a_2), "f"(pa_2)); - float _sub_565; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_565) : "f"(s_a_2), "f"(nm_t_5)); +#if __CUDA_ARCH__ == 1030 + float _sub_583; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(s_a_2), "f"(nm_t_5)); +#else + float _sub_569; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_569) : "f"(s_a_2), "f"(nm_t_5)); +#endif float nm_es_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_565), "f"(pa_2)); - float _sub_566; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_566) : "f"(pa_2), "f"(nm_t_5)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_583), "f"(pa_2)); + float _sub_584; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_584) : "f"(pa_2), "f"(nm_t_5)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_5) : "f"(_sub_569), "f"(pa_2)); + float _sub_570; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_570) : "f"(pa_2), "f"(nm_t_5)); +#endif float nm_ex_5; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_566), "f"(s_a_2)); - float _abs_568; - asm("abs.f32 %0, %1;" : "=f"(_abs_568) : "f"(s_a_2)); - float _abs_569; - asm("abs.f32 %0, %1;" : "=f"(_abs_569) : "f"(pa_2)); - uint32_t _setp_567_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_567_bits) : "f"(_abs_568), "f"(_abs_569)); - bool _setp_567 = _setp_567_bits != 0; - float nm_err_5 = ((_setp_567) ? nm_es_5 : nm_ex_5); - float _add_570; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_570) : "f"(c_a_2), "f"(nm_err_5)); - c_a_2 = _add_570; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_584), "f"(s_a_2)); + float _abs_586; + asm("abs.f32 %0, %1;" : "=f"(_abs_586) : "f"(s_a_2)); + float _abs_587; + asm("abs.f32 %0, %1;" : "=f"(_abs_587) : "f"(pa_2)); + uint32_t _setp_585_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_585_bits) : "f"(_abs_586), "f"(_abs_587)); + bool _setp_585 = _setp_585_bits != 0; + float nm_err_5 = ((_setp_585) ? nm_es_5 : nm_ex_5); + float _add_588; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_588) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_588; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_5) : "f"(_sub_570), "f"(s_a_2)); + float _abs_572; + asm("abs.f32 %0, %1;" : "=f"(_abs_572) : "f"(s_a_2)); + float _abs_573; + asm("abs.f32 %0, %1;" : "=f"(_abs_573) : "f"(pa_2)); + uint32_t _setp_571_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_571_bits) : "f"(_abs_572), "f"(_abs_573)); + bool _setp_571 = _setp_571_bits != 0; + float nm_err_5 = ((_setp_571) ? nm_es_5 : nm_ex_5); + float _add_574; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_574) : "f"(c_a_2), "f"(nm_err_5)); + c_a_2 = _add_574; +#endif s_a_2 = nm_t_5; float nm_t_6; asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_t_6) : "f"(s_b_2), "f"(pb_2)); - float _sub_571; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_571) : "f"(s_b_2), "f"(nm_t_6)); +#if __CUDA_ARCH__ == 1030 + float _sub_589; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(s_b_2), "f"(nm_t_6)); +#else + float _sub_575; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_575) : "f"(s_b_2), "f"(nm_t_6)); +#endif float nm_es_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_571), "f"(pb_2)); - float _sub_572; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_572) : "f"(pb_2), "f"(nm_t_6)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_589), "f"(pb_2)); + float _sub_590; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(pb_2), "f"(nm_t_6)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_es_6) : "f"(_sub_575), "f"(pb_2)); + float _sub_576; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_576) : "f"(pb_2), "f"(nm_t_6)); +#endif float nm_ex_6; - asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_572), "f"(s_b_2)); - float _abs_574; - asm("abs.f32 %0, %1;" : "=f"(_abs_574) : "f"(s_b_2)); - float _abs_575; - asm("abs.f32 %0, %1;" : "=f"(_abs_575) : "f"(pb_2)); - uint32_t _setp_573_bits; - asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_573_bits) : "f"(_abs_574), "f"(_abs_575)); - bool _setp_573 = _setp_573_bits != 0; - float nm_err_6 = ((_setp_573) ? nm_es_6 : nm_ex_6); - float _add_576; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_576) : "f"(c_b_2), "f"(nm_err_6)); - c_b_2 = _add_576; +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_590), "f"(s_b_2)); + float _abs_592; + asm("abs.f32 %0, %1;" : "=f"(_abs_592) : "f"(s_b_2)); + float _abs_593; + asm("abs.f32 %0, %1;" : "=f"(_abs_593) : "f"(pb_2)); + uint32_t _setp_591_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_591_bits) : "f"(_abs_592), "f"(_abs_593)); + bool _setp_591 = _setp_591_bits != 0; + float nm_err_6 = ((_setp_591) ? nm_es_6 : nm_ex_6); + float _add_594; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_594) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_594; +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(nm_ex_6) : "f"(_sub_576), "f"(s_b_2)); + float _abs_578; + asm("abs.f32 %0, %1;" : "=f"(_abs_578) : "f"(s_b_2)); + float _abs_579; + asm("abs.f32 %0, %1;" : "=f"(_abs_579) : "f"(pb_2)); + uint32_t _setp_577_bits; + asm("{ .reg .pred p; setp.ge.f32 p, %1, %2; selp.u32 %0, 1, 0, p; }" : "=r"(_setp_577_bits) : "f"(_abs_578), "f"(_abs_579)); + bool _setp_577 = _setp_577_bits != 0; + float nm_err_6 = ((_setp_577) ? nm_es_6 : nm_ex_6); + float _add_580; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_b_2), "f"(nm_err_6)); + c_b_2 = _add_580; +#endif s_b_2 = nm_t_6; gacc[0] = s_a_2; gacc[1] = c_a_2; @@ -6019,60 +8072,124 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s) : "f"(s_a_3), "f"(other[0])); float ts_bb; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb) : "f"(ts_s), "f"(s_a_3)); - float _sub_578; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_578) : "f"(ts_s), "f"(ts_bb)); - float _sub_577; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_577) : "f"(s_a_3), "f"(_sub_578)); - float _sub_579; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_579) : "f"(other[0]), "f"(ts_bb)); +#if __CUDA_ARCH__ == 1030 + float _sub_596; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_596) : "f"(ts_s), "f"(ts_bb)); + float _sub_595; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_595) : "f"(s_a_3), "f"(_sub_596)); + float _sub_597; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_597) : "f"(other[0]), "f"(ts_bb)); +#else + float _sub_582; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s), "f"(ts_bb)); + float _sub_581; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(s_a_3), "f"(_sub_582)); + float _sub_583; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(other[0]), "f"(ts_bb)); +#endif float ts_e; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_577), "f"(_sub_579)); - float _add_580; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_580) : "f"(c_a_3), "f"(other[1])); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_595), "f"(_sub_597)); + float _add_598; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_598) : "f"(c_a_3), "f"(other[1])); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e) : "f"(_sub_581), "f"(_sub_583)); + float _add_584; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_584) : "f"(c_a_3), "f"(other[1])); +#endif float pm_c; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_580), "f"(ts_e)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_598), "f"(ts_e)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c) : "f"(_add_584), "f"(ts_e)); +#endif float ts_s_0; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_0) : "f"(ts_s), "f"(pm_c)); float ts_bb_0; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_0) : "f"(ts_s_0), "f"(ts_s)); - float _sub_582; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_582) : "f"(ts_s_0), "f"(ts_bb_0)); - float _sub_581; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_581) : "f"(ts_s), "f"(_sub_582)); - float _sub_583; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_583) : "f"(pm_c), "f"(ts_bb_0)); +#if __CUDA_ARCH__ == 1030 + float _sub_600; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_600) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_599; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_599) : "f"(ts_s), "f"(_sub_600)); + float _sub_601; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_601) : "f"(pm_c), "f"(ts_bb_0)); +#else + float _sub_586; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(ts_s_0), "f"(ts_bb_0)); + float _sub_585; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s), "f"(_sub_586)); + float _sub_587; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_587) : "f"(pm_c), "f"(ts_bb_0)); +#endif float ts_e_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_581), "f"(_sub_583)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_599), "f"(_sub_601)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_0) : "f"(_sub_585), "f"(_sub_587)); +#endif s_a_3 = ts_s_0; c_a_3 = ts_e_0; float ts_s_1; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_1) : "f"(s_b_3), "f"(other[2])); float ts_bb_1; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_1) : "f"(ts_s_1), "f"(s_b_3)); - float _sub_585; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_585) : "f"(ts_s_1), "f"(ts_bb_1)); - float _sub_584; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_584) : "f"(s_b_3), "f"(_sub_585)); - float _sub_586; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_586) : "f"(other[2]), "f"(ts_bb_1)); +#if __CUDA_ARCH__ == 1030 + float _sub_603; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_603) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_602; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_602) : "f"(s_b_3), "f"(_sub_603)); + float _sub_604; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_604) : "f"(other[2]), "f"(ts_bb_1)); +#else + float _sub_589; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_1), "f"(ts_bb_1)); + float _sub_588; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(s_b_3), "f"(_sub_589)); + float _sub_590; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(other[2]), "f"(ts_bb_1)); +#endif float ts_e_1; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_584), "f"(_sub_586)); - float _add_587; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_587) : "f"(c_b_3), "f"(other[3])); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_602), "f"(_sub_604)); + float _add_605; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_605) : "f"(c_b_3), "f"(other[3])); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_1) : "f"(_sub_588), "f"(_sub_590)); + float _add_591; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_591) : "f"(c_b_3), "f"(other[3])); +#endif float pm_c_0; - asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_587), "f"(ts_e_1)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_605), "f"(ts_e_1)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(pm_c_0) : "f"(_add_591), "f"(ts_e_1)); +#endif float ts_s_2; asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_s_2) : "f"(ts_s_1), "f"(pm_c_0)); float ts_bb_2; asm("sub.rn.f32 %0, %1, %2;" : "=f"(ts_bb_2) : "f"(ts_s_2), "f"(ts_s_1)); - float _sub_589; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_589) : "f"(ts_s_2), "f"(ts_bb_2)); - float _sub_588; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_588) : "f"(ts_s_1), "f"(_sub_589)); - float _sub_590; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_590) : "f"(pm_c_0), "f"(ts_bb_2)); +#if __CUDA_ARCH__ == 1030 + float _sub_607; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_607) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_606; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_606) : "f"(ts_s_1), "f"(_sub_607)); + float _sub_608; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_608) : "f"(pm_c_0), "f"(ts_bb_2)); +#else + float _sub_593; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_593) : "f"(ts_s_2), "f"(ts_bb_2)); + float _sub_592; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_592) : "f"(ts_s_1), "f"(_sub_593)); + float _sub_594; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_594) : "f"(pm_c_0), "f"(ts_bb_2)); +#endif float ts_e_2; - asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_588), "f"(_sub_590)); +#if __CUDA_ARCH__ == 1030 + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_606), "f"(_sub_608)); +#else + asm("add.rn.f32 %0, %1, %2;" : "=f"(ts_e_2) : "f"(_sub_592), "f"(_sub_594)); +#endif s_b_3 = ts_s_2; c_b_3 = ts_e_2; double v_a = (double)s_a_3 + (double)c_a_3; @@ -6175,41 +8292,79 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d z[2] = 0.0f; z[3] = 0.0f; #pragma unroll 1 - for (int r_347 = 0; r_347 < 16; r_347++) { - int t = r_347; - if (t >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_347 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); +#if __CUDA_ARCH__ == 1030 + for (int r_326 = 0; r_326 < 16; r_326++) { + int t_24 = r_326; + if (t_24 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_24 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_24 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_326 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); +#else + for (int r_305 = 0; r_305 < 16; r_305++) { + int t_16 = r_305; + if (t_16 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 16)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 32)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 48)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 64)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 80)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 96)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_16 * 128 + 112)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_305 * 128)), "f"(z[0]), "f"(z[1]), "f"(z[2]), "f"(z[3]) : "memory"); +#endif } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6276,41 +8431,79 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d z_1[2] = 0.0f; z_1[3] = 0.0f; #pragma unroll 1 - for (int r_348 = 0; r_348 < 16; r_348++) { - int t_32 = 16 + r_348; - if (t_32 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_32 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_32 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_348 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); +#if __CUDA_ARCH__ == 1030 + for (int r_327 = 0; r_327 < 16; r_327++) { + int t_25 = 16 + r_327; + if (t_25 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_25 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_25 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_327 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); +#else + for (int r_306 = 0; r_306 < 16; r_306++) { + int t_17 = 16 + r_306; + if (t_17 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 16)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 32)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 48)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 64)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 80)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 96)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_17 * 128 + 112)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_306 * 128)), "f"(z_1[0]), "f"(z_1[1]), "f"(z_1[2]), "f"(z_1[3]) : "memory"); +#endif } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6416,41 +8609,79 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d z_2[2] = 0.0f; z_2[3] = 0.0f; #pragma unroll 1 - for (int r_349 = 0; r_349 < 16; r_349++) { - int t_33 = 32 + r_349; - if (t_33 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_33 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_33 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_349 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); +#if __CUDA_ARCH__ == 1030 + for (int r_328 = 0; r_328 < 16; r_328++) { + int t_26 = 32 + r_328; + if (t_26 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_26 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_26 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_328 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); +#else + for (int r_307 = 0; r_307 < 16; r_307++) { + int t_18 = 32 + r_307; + if (t_18 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98304 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98320 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98336 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(98416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100352 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 16)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100368 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 32)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100384 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 48)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100400 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 64)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100416 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 80)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100432 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 96)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100448 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_18 * 128 + 112)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(100464 + r_307 * 128)), "f"(z_2[0]), "f"(z_2[1]), "f"(z_2[2]), "f"(z_2[3]) : "memory"); +#endif } } asm volatile("tcgen05.fence::before_thread_sync;"); @@ -6583,41 +8814,79 @@ kernel_cake_kda_train_87b73f813c841159bf43(const __grid_constant__ CUtensorMap d z_3[2] = 0.0f; z_3[3] = 0.0f; #pragma unroll 1 - for (int r_350 = 0; r_350 < 16; r_350++) { - int t_34 = 48 + r_350; - if (t_34 >= clen_setup) { - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_34 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_34 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); - asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_350 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); +#if __CUDA_ARCH__ == 1030 + for (int r_329 = 0; r_329 < 16; r_329++) { + int t_27 = 48 + r_329; + if (t_27 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_27 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_27 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_329 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); +#else + for (int r_308 = 0; r_308 < 16; r_308++) { + int t_19 = 48 + r_308; + if (t_19 >= clen_setup) { + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102400 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102416 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102432 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(32768 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(102512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104448 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 16)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104464 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 32)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104480 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 48)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104496 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 64)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104512 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 80)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104528 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 96)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104544 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(40960 + t_19 * 128 + 112)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); + asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(pool_f32_addr + (unsigned int)(104560 + r_308 * 128)), "f"(z_3[0]), "f"(z_3[1]), "f"(z_3[2]), "f"(z_3[3]) : "memory"); +#endif } } asm volatile("tcgen05.fence::before_thread_sync;"); diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu index af6add58ac8..59b5fcea001 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_8ce96eeb5b7517229400_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2246,15 +2269,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2275,15 +2304,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2304,15 +2339,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2333,15 +2374,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2502,15 +2573,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2531,15 +2608,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2560,15 +2643,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2589,15 +2678,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2636,15 +2731,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2665,15 +2766,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2694,15 +2801,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2723,15 +2836,21 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2802,37 +2921,37 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2967,7 +3086,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[7]; float q_r_0 = q_t0_0; - float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, (dk2b_0 - dkt_3[7]) * k_t_0)) + dgf_0; + float qe = q_r_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[7], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[7]); + float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[7], k_t_0, cr))) + dgf_0; s_run[0] = s_run[0] + dg_o_0; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -2977,14 +3101,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_248 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_249 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 6; @@ -3011,7 +3135,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[6]; float q_r_1 = q_t0_1; - float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, (dk2b_1 - dkt_3[6]) * k_t_1)) + dgf_1; + float qe_0 = q_r_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[6], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[6]); + float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[6], k_t_1, cr_0))) + dgf_1; s_run[0] = s_run[0] + dg_o_1; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3021,14 +3150,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_1 = t_1 * qrs; if (t_1 < clen) { { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_250 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_251 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 5; @@ -3055,7 +3184,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[5]; float q_r_2 = q_t0_2; - float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, (dk2b_2 - dkt_3[5]) * k_t_2)) + dgf_2; + float qe_1 = q_r_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[5], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[5]); + float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[5], k_t_2, cr_1))) + dgf_2; s_run[0] = s_run[0] + dg_o_2; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3065,14 +3199,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_252 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_253 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 4; @@ -3099,7 +3233,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[4]; float q_r_3 = q_t0_3; - float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, (dk2b_3 - dkt_3[4]) * k_t_3)) + dgf_3; + float qe_2 = q_r_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[4], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[4]); + float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[4], k_t_3, cr_2))) + dgf_3; s_run[0] = s_run[0] + dg_o_3; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3109,14 +3248,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_254 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_255 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 3; @@ -3143,7 +3282,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[3]; float q_r_4 = q_t0_4; - float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, (dk2b_4 - dkt_3[3]) * k_t_4)) + dgf_4; + float qe_3 = q_r_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[3], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[3]); + float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[3], k_t_4, cr_3))) + dgf_4; s_run[0] = s_run[0] + dg_o_4; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3153,14 +3297,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_256 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_257 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 2; @@ -3187,7 +3331,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[2]; float q_r_5 = q_t0_5; - float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, (dk2b_5 - dkt_3[2]) * k_t_5)) + dgf_5; + float qe_4 = q_r_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[2], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[2]); + float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[2], k_t_5, cr_4))) + dgf_5; s_run[0] = s_run[0] + dg_o_5; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3197,14 +3346,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_258 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_259 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 1; @@ -3231,7 +3380,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[1]; float q_r_6 = q_t0_6; - float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, (dk2b_6 - dkt_3[1]) * k_t_6)) + dgf_6; + float qe_5 = q_r_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[1], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[1]); + float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[1], k_t_6, cr_5))) + dgf_6; s_run[0] = s_run[0] + dg_o_6; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3241,14 +3395,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_260 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_261 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th; @@ -3275,7 +3429,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[0]; float q_r_7 = q_t0_7; - float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, (dk2b_7 - dkt_3[0]) * k_t_7)) + dgf_7; + float qe_6 = q_r_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[0], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[0]); + float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[0], k_t_7, cr_6))) + dgf_7; s_run[0] = s_run[0] + dg_o_7; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3285,14 +3444,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_262 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_263 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_7; } } float tot = s_run[0]; @@ -3301,31 +3460,31 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3386,7 +3545,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[7]; float q_r_8 = q_t0_8; - float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, (dk2b_8 - dkt_4[7]) * k_t_8)) + dgf_8; + float qe_7 = q_r_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[7], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[7]); + float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[7], k_t_8, cr_7))) + dgf_8; s_run_0[0] = s_run_0[0] + dg_o_8; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3396,14 +3560,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_273 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_274 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 6; @@ -3430,7 +3594,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[6]; float q_r_9 = q_t0_9; - float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, (dk2b_9 - dkt_4[6]) * k_t_9)) + dgf_9; + float qe_8 = q_r_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[6], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[6]); + float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[6], k_t_9, cr_8))) + dgf_9; s_run_0[0] = s_run_0[0] + dg_o_9; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3440,14 +3609,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_9 = t_9 * qrs; if (t_9 < clen) { { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_275 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_276 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 5; @@ -3474,7 +3643,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[5]; float q_r_10 = q_t0_10; - float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, (dk2b_10 - dkt_4[5]) * k_t_10)) + dgf_10; + float qe_9 = q_r_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[5], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[5]); + float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[5], k_t_10, cr_9))) + dgf_10; s_run_0[0] = s_run_0[0] + dg_o_10; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3484,14 +3658,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_10 = t_10 * qrs; if (t_10 < clen) { { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_277 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_278 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 4; @@ -3518,7 +3692,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[4]; float q_r_11 = q_t0_11; - float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, (dk2b_11 - dkt_4[4]) * k_t_11)) + dgf_11; + float qe_10 = q_r_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[4], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[4]); + float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[4], k_t_11, cr_10))) + dgf_11; s_run_0[0] = s_run_0[0] + dg_o_11; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3528,14 +3707,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_11 = t_11 * qrs; if (t_11 < clen) { { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_279 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_280 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 3; @@ -3562,7 +3741,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[3]; float q_r_12 = q_t0_12; - float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, (dk2b_12 - dkt_4[3]) * k_t_12)) + dgf_12; + float qe_11 = q_r_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[3], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[3]); + float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[3], k_t_12, cr_11))) + dgf_12; s_run_0[0] = s_run_0[0] + dg_o_12; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3572,14 +3756,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_12 = t_12 * qrs; if (t_12 < clen) { { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_281 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_282 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 2; @@ -3606,7 +3790,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[2]; float q_r_13 = q_t0_13; - float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, (dk2b_13 - dkt_4[2]) * k_t_13)) + dgf_13; + float qe_12 = q_r_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[2], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[2]); + float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[2], k_t_13, cr_12))) + dgf_13; s_run_0[0] = s_run_0[0] + dg_o_13; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3616,14 +3805,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_13 = t_13 * qrs; if (t_13 < clen) { { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_283 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_284 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 1; @@ -3650,7 +3839,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[1]; float q_r_14 = q_t0_14; - float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, (dk2b_14 - dkt_4[1]) * k_t_14)) + dgf_14; + float qe_13 = q_r_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[1], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[1]); + float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[1], k_t_14, cr_13))) + dgf_14; s_run_0[0] = s_run_0[0] + dg_o_14; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3660,14 +3854,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_14 = t_14 * qrs; if (t_14 < clen) { { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_285 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_286 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th; @@ -3694,7 +3888,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[0]; float q_r_15 = q_t0_15; - float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, (dk2b_15 - dkt_4[0]) * k_t_15)) + dgf_15; + float qe_14 = q_r_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[0], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[0]); + float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[0], k_t_15, cr_14))) + dgf_15; s_run_0[0] = s_run_0[0] + dg_o_15; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -3704,14 +3903,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_15 = t_15 * qrs; if (t_15 < clen) { { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_287 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_288 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_15; } } float tot_0 = s_run_0[0]; @@ -3720,36 +3919,36 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_289; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_290; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_291; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_292; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_293; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_294; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -3778,9 +3977,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_0 = offset + sfx[0]; if (t_16 < clen) { { - auto _gst_267 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_0; + auto _gst_299 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_0; } } int t_17 = 48 + 8 * th + 1; @@ -3788,9 +3987,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_1 = offset + sfx[1]; if (t_17 < clen) { { - auto _gst_268 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1; + auto _gst_300 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1; } } int t_18 = 48 + 8 * th + 2; @@ -3798,9 +3997,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_2 = offset + sfx[2]; if (t_18 < clen) { { - auto _gst_269 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_2; + auto _gst_301 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_2; } } int t_19 = 48 + 8 * th + 3; @@ -3808,9 +4007,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_3 = offset + sfx[3]; if (t_19 < clen) { { - auto _gst_270 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_3; + auto _gst_302 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_3; } } int t_20 = 48 + 8 * th + 4; @@ -3818,9 +4017,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_4 = offset + sfx[4]; if (t_20 < clen) { { - auto _gst_271 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_4; + auto _gst_303 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_4; } } int t_21 = 48 + 8 * th + 5; @@ -3828,9 +4027,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_5 = offset + sfx[5]; if (t_21 < clen) { { - auto _gst_272 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_5; + auto _gst_304 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_5; } } int t_22 = 48 + 8 * th + 6; @@ -3838,9 +4037,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_6 = offset + sfx[6]; if (t_22 < clen) { { - auto _gst_273 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_6; + auto _gst_305 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_6; } } int t_23 = 48 + 8 * th + 7; @@ -3848,9 +4047,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_7 = offset + sfx[7]; if (t_23 < clen) { { - auto _gst_274 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_7; + auto _gst_306 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3868,9 +4067,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_8 = offset_0 + sfx_0[0]; if (t_24 < clen) { { - auto _gst_275 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_8; + auto _gst_307 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_8; } } int t_25 = 32 + 8 * th + 1; @@ -3878,9 +4077,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_9 = offset_0 + sfx_0[1]; if (t_25 < clen) { { - auto _gst_276 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_9; + auto _gst_308 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_9; } } int t_26 = 32 + 8 * th + 2; @@ -3888,9 +4087,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_10 = offset_0 + sfx_0[2]; if (t_26 < clen) { { - auto _gst_277 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_10; + auto _gst_309 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_10; } } int t_27 = 32 + 8 * th + 3; @@ -3898,9 +4097,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_11 = offset_0 + sfx_0[3]; if (t_27 < clen) { { - auto _gst_278 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_11; + auto _gst_310 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_11; } } int t_28 = 32 + 8 * th + 4; @@ -3908,9 +4107,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_12 = offset_0 + sfx_0[4]; if (t_28 < clen) { { - auto _gst_279 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_12; + auto _gst_311 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_12; } } int t_29 = 32 + 8 * th + 5; @@ -3918,9 +4117,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_13 = offset_0 + sfx_0[5]; if (t_29 < clen) { { - auto _gst_280 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_13; + auto _gst_312 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_13; } } int t_30 = 32 + 8 * th + 6; @@ -3928,9 +4127,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_14 = offset_0 + sfx_0[6]; if (t_30 < clen) { { - auto _gst_281 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_14; + auto _gst_313 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_14; } } int t_31 = 32 + 8 * th + 7; @@ -3938,15 +4137,15 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_15 = offset_0 + sfx_0[7]; if (t_31 < clen) { { - auto _gst_282 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_15; + auto _gst_314 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4007,7 +4206,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[7]; float q_r_16 = q_t0_16; - float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, (dk2b_16 - dkt_5[7]) * k_t_16)) + dgf_16; + float qe_15 = q_r_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[7], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[7]); + float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[7], k_t_16, cr_15))) + dgf_16; s_run_1[0] = s_run_1[0] + dg_o_16; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4017,14 +4221,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_284 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_316 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_285 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_317 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_33 = 16 + 8 * th + 6; @@ -4051,7 +4255,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[6]; float q_r_17 = q_t0_17; - float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, (dk2b_17 - dkt_5[6]) * k_t_17)) + dgf_17; + float qe_16 = q_r_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[6], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[6]); + float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[6], k_t_17, cr_16))) + dgf_17; s_run_1[0] = s_run_1[0] + dg_o_17; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4061,14 +4270,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_17 = t_33 * qrs; if (t_33 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_34 = 16 + 8 * th + 5; @@ -4095,7 +4304,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[5]; float q_r_18 = q_t0_18; - float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, (dk2b_18 - dkt_5[5]) * k_t_18)) + dgf_18; + float qe_17 = q_r_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[5], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[5]); + float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[5], k_t_18, cr_17))) + dgf_18; s_run_1[0] = s_run_1[0] + dg_o_18; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4105,14 +4319,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_18 = t_34 * qrs; if (t_34 < clen) { { - auto _gst_288 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_320 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_289 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_321 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_35 = 16 + 8 * th + 4; @@ -4139,7 +4353,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[4]; float q_r_19 = q_t0_19; - float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, (dk2b_19 - dkt_5[4]) * k_t_19)) + dgf_19; + float qe_18 = q_r_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[4], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[4]); + float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[4], k_t_19, cr_18))) + dgf_19; s_run_1[0] = s_run_1[0] + dg_o_19; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4149,14 +4368,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_19 = t_35 * qrs; if (t_35 < clen) { { - auto _gst_290 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_322 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_291 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_323 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_36 = 16 + 8 * th + 3; @@ -4183,7 +4402,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[3]; float q_r_20 = q_t0_20; - float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, (dk2b_20 - dkt_5[3]) * k_t_20)) + dgf_20; + float qe_19 = q_r_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[3], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[3]); + float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[3], k_t_20, cr_19))) + dgf_20; s_run_1[0] = s_run_1[0] + dg_o_20; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4193,14 +4417,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_20 = t_36 * qrs; if (t_36 < clen) { { - auto _gst_292 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_324 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_293 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_325 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_37 = 16 + 8 * th + 2; @@ -4227,7 +4451,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[2]; float q_r_21 = q_t0_21; - float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, (dk2b_21 - dkt_5[2]) * k_t_21)) + dgf_21; + float qe_20 = q_r_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[2], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[2]); + float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[2], k_t_21, cr_20))) + dgf_21; s_run_1[0] = s_run_1[0] + dg_o_21; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4237,14 +4466,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_21 = t_37 * qrs; if (t_37 < clen) { { - auto _gst_294 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_326 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_295 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_327 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_38 = 16 + 8 * th + 1; @@ -4271,7 +4500,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[1]; float q_r_22 = q_t0_22; - float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, (dk2b_22 - dkt_5[1]) * k_t_22)) + dgf_22; + float qe_21 = q_r_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[1], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[1]); + float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[1], k_t_22, cr_21))) + dgf_22; s_run_1[0] = s_run_1[0] + dg_o_22; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4281,14 +4515,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_22 = t_38 * qrs; if (t_38 < clen) { { - auto _gst_296 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_328 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_297 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_329 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_39 = 16 + 8 * th; @@ -4315,7 +4549,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[0]; float q_r_23 = q_t0_23; - float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, (dk2b_23 - dkt_5[0]) * k_t_23)) + dgf_23; + float qe_22 = q_r_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[0], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[0]); + float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[0], k_t_23, cr_22))) + dgf_23; s_run_1[0] = s_run_1[0] + dg_o_23; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4325,14 +4564,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_23 = t_39 * qrs; if (t_39 < clen) { { - auto _gst_298 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_330 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_299 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_331 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_23; } } float tot_1 = s_run_1[0]; @@ -4341,37 +4580,37 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_332; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_333; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_334; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_335; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_336; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_337; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4400,9 +4639,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_16 = offset_1 + sfx_1[0]; if (t_40 < clen) { { - auto _gst_310 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_16; + auto _gst_342 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_16; } } int t_41 = 16 + 8 * th + 1; @@ -4410,9 +4649,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_17 = offset_1 + sfx_1[1]; if (t_41 < clen) { { - auto _gst_311 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_17; + auto _gst_343 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_17; } } int t_42 = 16 + 8 * th + 2; @@ -4420,9 +4659,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_18 = offset_1 + sfx_1[2]; if (t_42 < clen) { { - auto _gst_312 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_18; + auto _gst_344 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_18; } } int t_43 = 16 + 8 * th + 3; @@ -4430,9 +4669,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_19 = offset_1 + sfx_1[3]; if (t_43 < clen) { { - auto _gst_313 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_19; + auto _gst_345 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_19; } } int t_44 = 16 + 8 * th + 4; @@ -4440,9 +4679,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_20 = offset_1 + sfx_1[4]; if (t_44 < clen) { { - auto _gst_314 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_20; + auto _gst_346 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_20; } } int t_45 = 16 + 8 * th + 5; @@ -4450,9 +4689,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_21 = offset_1 + sfx_1[5]; if (t_45 < clen) { { - auto _gst_315 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_21; + auto _gst_347 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_21; } } int t_46 = 16 + 8 * th + 6; @@ -4460,9 +4699,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_22 = offset_1 + sfx_1[6]; if (t_46 < clen) { { - auto _gst_316 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_22; + auto _gst_348 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_22; } } int t_47 = 16 + 8 * th + 7; @@ -4470,15 +4709,15 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_23 = offset_1 + sfx_1[7]; if (t_47 < clen) { { - auto _gst_317 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_23; + auto _gst_349 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4533,7 +4772,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[7]; float q_r_24 = q_t0_24; - float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, (dk2b_24 - dkt_6[7]) * k_t_24)) + dgf_24; + float qe_23 = q_r_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[7], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[7]); + float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[7], k_t_24, cr_23))) + dgf_24; s_run_2[0] = s_run_2[0] + dg_o_24; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4543,14 +4787,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_319 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_351 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_320 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_352 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_49 = 8 * th + 6; @@ -4577,7 +4821,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[6]; float q_r_25 = q_t0_25; - float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, (dk2b_25 - dkt_6[6]) * k_t_25)) + dgf_25; + float qe_24 = q_r_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[6], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[6]); + float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[6], k_t_25, cr_24))) + dgf_25; s_run_2[0] = s_run_2[0] + dg_o_25; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4587,14 +4836,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_25 = t_49 * qrs; if (t_49 < clen) { { - auto _gst_321 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_353 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_322 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_354 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_50 = 8 * th + 5; @@ -4621,7 +4870,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[5]; float q_r_26 = q_t0_26; - float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, (dk2b_26 - dkt_6[5]) * k_t_26)) + dgf_26; + float qe_25 = q_r_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[5], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[5]); + float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[5], k_t_26, cr_25))) + dgf_26; s_run_2[0] = s_run_2[0] + dg_o_26; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4631,14 +4885,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_26 = t_50 * qrs; if (t_50 < clen) { { - auto _gst_323 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_355 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_324 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_356 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_51 = 8 * th + 4; @@ -4665,7 +4919,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[4]; float q_r_27 = q_t0_27; - float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, (dk2b_27 - dkt_6[4]) * k_t_27)) + dgf_27; + float qe_26 = q_r_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[4], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[4]); + float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[4], k_t_27, cr_26))) + dgf_27; s_run_2[0] = s_run_2[0] + dg_o_27; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4675,14 +4934,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_27 = t_51 * qrs; if (t_51 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_52 = 8 * th + 3; @@ -4709,7 +4968,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[3]; float q_r_28 = q_t0_28; - float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, (dk2b_28 - dkt_6[3]) * k_t_28)) + dgf_28; + float qe_27 = q_r_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[3], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[3]); + float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[3], k_t_28, cr_27))) + dgf_28; s_run_2[0] = s_run_2[0] + dg_o_28; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4719,14 +4983,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_28 = t_52 * qrs; if (t_52 < clen) { { - auto _gst_327 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_359 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_328 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_360 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_53 = 8 * th + 2; @@ -4753,7 +5017,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[2]; float q_r_29 = q_t0_29; - float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, (dk2b_29 - dkt_6[2]) * k_t_29)) + dgf_29; + float qe_28 = q_r_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[2], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[2]); + float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[2], k_t_29, cr_28))) + dgf_29; s_run_2[0] = s_run_2[0] + dg_o_29; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4763,14 +5032,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_29 = t_53 * qrs; if (t_53 < clen) { { - auto _gst_329 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_361 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_330 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_362 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_54 = 8 * th + 1; @@ -4797,7 +5066,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[1]; float q_r_30 = q_t0_30; - float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, (dk2b_30 - dkt_6[1]) * k_t_30)) + dgf_30; + float qe_29 = q_r_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[1], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[1]); + float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[1], k_t_30, cr_29))) + dgf_30; s_run_2[0] = s_run_2[0] + dg_o_30; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4807,14 +5081,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_30 = t_54 * qrs; if (t_54 < clen) { { - auto _gst_331 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_363 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_332 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_364 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_55 = 8 * th; @@ -4841,7 +5115,12 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[0]; float q_r_31 = q_t0_31; - float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, (dk2b_31 - dkt_6[0]) * k_t_31)) + dgf_31; + float qe_30 = q_r_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[0], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[0]); + float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[0], k_t_31, cr_30))) + dgf_31; s_run_2[0] = s_run_2[0] + dg_o_31; asm volatile( "tcgen05.st.sync.aligned.32x32b.x1.b32" @@ -4851,14 +5130,14 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int qoff_31 = t_55 * qrs; if (t_55 < clen) { { - auto _gst_333 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_365 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_334 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_366 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_31; } } float tot_2 = s_run_2[0]; @@ -4867,35 +5146,35 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_367; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_368; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_369; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_370; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_371; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_372; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_2) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); if (qd == 0) { @@ -4924,9 +5203,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_24 = offset_2 + sfx_2[0]; if (t_56 < clen) { { - auto _gst_345 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_24; + auto _gst_377 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_24; } } int t_57 = 8 * th + 1; @@ -4934,9 +5213,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_25 = offset_2 + sfx_2[1]; if (t_57 < clen) { { - auto _gst_346 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_25; + auto _gst_378 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_25; } } int t_58 = 8 * th + 2; @@ -4944,9 +5223,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_26 = offset_2 + sfx_2[2]; if (t_58 < clen) { { - auto _gst_347 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_26; + auto _gst_379 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_26; } } int t_59 = 8 * th + 3; @@ -4954,9 +5233,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_27 = offset_2 + sfx_2[3]; if (t_59 < clen) { { - auto _gst_348 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_27; + auto _gst_380 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_27; } } int t_60 = 8 * th + 4; @@ -4964,9 +5243,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_28 = offset_2 + sfx_2[4]; if (t_60 < clen) { { - auto _gst_349 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_28; + auto _gst_381 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_28; } } int t_61 = 8 * th + 5; @@ -4974,9 +5253,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_29 = offset_2 + sfx_2[5]; if (t_61 < clen) { { - auto _gst_350 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_29; + auto _gst_382 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_29; } } int t_62 = 8 * th + 6; @@ -4984,9 +5263,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_30 = offset_2 + sfx_2[6]; if (t_62 < clen) { { - auto _gst_351 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_30; + auto _gst_383 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_30; } } int t_63 = 8 * th + 7; @@ -4994,9 +5273,9 @@ kernel_cake_kda_train_8ce96eeb5b7517229400(const __grid_constant__ CUtensorMap d float dg_31 = offset_2 + sfx_2[7]; if (t_63 < clen) { { - auto _gst_352 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_31; + auto _gst_384 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_31; } } off_acc[0] = off_acc[0] + t_hi_2 + t_lo_2; diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu index a07b0c5e214..95e9120961d 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_d5fd3f7234a4266e09d2_kernel.cu @@ -130,6 +130,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -172,6 +179,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2235,15 +2258,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2264,15 +2293,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2293,15 +2328,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2322,15 +2363,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2366,12 +2413,18 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2391,12 +2444,18 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2416,12 +2475,18 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2441,12 +2506,18 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2491,15 +2562,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2520,15 +2597,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2549,15 +2632,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2578,15 +2667,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2625,15 +2720,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2654,15 +2755,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2683,15 +2790,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2712,15 +2825,21 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2791,37 +2910,37 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2833,29 +2952,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2863,29 +2982,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2954,24 +3073,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; float q_r_0 = q_t0_0; - float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0)) + dgf_0; + float qe = q_r_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr))) + dgf_0; int goff_0 = t_0 * rs_g; int qoff_0 = t_0 * qrs; { - auto _gst_216 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast(_gst_216) + (0)) = dg_o_0; + auto _gst_248 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast(_gst_248) + (0)) = dg_o_0; } if (t_0 < clen) { { - auto _gst_217 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_249 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_218 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_250 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -2998,24 +3122,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; float q_r_1 = q_t0_1; - float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1)) + dgf_1; + float qe_0 = q_r_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0))) + dgf_1; int goff_1 = t_1 * rs_g; int qoff_1 = t_1 * qrs; { - auto _gst_219 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast(_gst_219) + (0)) = dg_o_1; + auto _gst_251 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast(_gst_251) + (0)) = dg_o_1; } if (t_1 < clen) { { - auto _gst_220 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_252 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_221 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_253 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3042,24 +3171,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; float q_r_2 = q_t0_2; - float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2)) + dgf_2; + float qe_1 = q_r_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1))) + dgf_2; int goff_2 = t_2 * rs_g; int qoff_2 = t_2 * qrs; { - auto _gst_222 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast(_gst_222) + (0)) = dg_o_2; + auto _gst_254 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast(_gst_254) + (0)) = dg_o_2; } if (t_2 < clen) { { - auto _gst_223 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_255 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_224 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_256 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3086,24 +3220,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; float q_r_3 = q_t0_3; - float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3)) + dgf_3; + float qe_2 = q_r_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2))) + dgf_3; int goff_3 = t_3 * rs_g; int qoff_3 = t_3 * qrs; { - auto _gst_225 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast(_gst_225) + (0)) = dg_o_3; + auto _gst_257 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast(_gst_257) + (0)) = dg_o_3; } if (t_3 < clen) { { - auto _gst_226 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_258 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_227 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_259 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3130,24 +3269,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; float q_r_4 = q_t0_4; - float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4)) + dgf_4; + float qe_3 = q_r_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3))) + dgf_4; int goff_4 = t_4 * rs_g; int qoff_4 = t_4 * qrs; { - auto _gst_228 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast(_gst_228) + (0)) = dg_o_4; + auto _gst_260 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast(_gst_260) + (0)) = dg_o_4; } if (t_4 < clen) { { - auto _gst_229 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_261 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_230 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_262 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3174,24 +3318,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; float q_r_5 = q_t0_5; - float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5)) + dgf_5; + float qe_4 = q_r_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4))) + dgf_5; int goff_5 = t_5 * rs_g; int qoff_5 = t_5 * qrs; { - auto _gst_231 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast(_gst_231) + (0)) = dg_o_5; + auto _gst_263 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast(_gst_263) + (0)) = dg_o_5; } if (t_5 < clen) { { - auto _gst_232 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_232)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_232) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_264 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_264)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_264) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_233 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_233)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_233) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_265 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_265)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3218,24 +3367,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; float q_r_6 = q_t0_6; - float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6)) + dgf_6; + float qe_5 = q_r_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5))) + dgf_6; int goff_6 = t_6 * rs_g; int qoff_6 = t_6 * qrs; { - auto _gst_234 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_234)); - *(reinterpret_cast(_gst_234) + (0)) = dg_o_6; + auto _gst_266 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_266)); + *(reinterpret_cast(_gst_266) + (0)) = dg_o_6; } if (t_6 < clen) { { - auto _gst_235 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_235)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_235) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_267 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_267)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_267) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_236 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_236)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_236) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_268 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_268)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3262,24 +3416,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; float q_r_7 = q_t0_7; - float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7)) + dgf_7; + float qe_6 = q_r_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6))) + dgf_7; int goff_7 = t_7 * rs_g; int qoff_7 = t_7 * qrs; { - auto _gst_237 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_237)); - *(reinterpret_cast(_gst_237) + (0)) = dg_o_7; + auto _gst_269 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_269)); + *(reinterpret_cast(_gst_269) + (0)) = dg_o_7; } if (t_7 < clen) { { - auto _gst_238 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_238)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_238) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_270 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_270)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_270) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_239 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_239)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_239) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_271 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_271)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dk_o_7; } } int b4 = lane_1 >> 4 & 1; @@ -3287,31 +3446,31 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_240; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_272; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_241 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_241; + float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_273; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_242 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_242; + float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_274; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_243 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_243; + float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_275; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_244 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_244; + float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_276; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_245 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_245; + float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_277; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_246 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_246; - float _shfl_xor_sync_247 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_247; - float _shfl_xor_sync_248 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_248; + float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_278; + float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_279; + float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_280; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3370,24 +3529,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; float q_r_8 = q_t0_8; - float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8)) + dgf_8; + float qe_7 = q_r_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7))) + dgf_8; int goff_8 = t_8 * rs_g; int qoff_8 = t_8 * qrs; { - auto _gst_249 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast(_gst_249) + (0)) = dg_o_8; + auto _gst_281 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast(_gst_281) + (0)) = dg_o_8; } if (t_8 < clen) { { - auto _gst_250 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_282 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_251 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_283 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3414,24 +3578,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; float q_r_9 = q_t0_9; - float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9)) + dgf_9; + float qe_8 = q_r_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8))) + dgf_9; int goff_9 = t_9 * rs_g; int qoff_9 = t_9 * qrs; { - auto _gst_252 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast(_gst_252) + (0)) = dg_o_9; + auto _gst_284 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast(_gst_284) + (0)) = dg_o_9; } if (t_9 < clen) { { - auto _gst_253 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_285 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_254 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_286 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3458,24 +3627,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; float q_r_10 = q_t0_10; - float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10)) + dgf_10; + float qe_9 = q_r_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9))) + dgf_10; int goff_10 = t_10 * rs_g; int qoff_10 = t_10 * qrs; { - auto _gst_255 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast(_gst_255) + (0)) = dg_o_10; + auto _gst_287 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast(_gst_287) + (0)) = dg_o_10; } if (t_10 < clen) { { - auto _gst_256 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_288 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_257 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_257)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_289 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_289)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3502,24 +3676,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; float q_r_11 = q_t0_11; - float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11)) + dgf_11; + float qe_10 = q_r_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10))) + dgf_11; int goff_11 = t_11 * rs_g; int qoff_11 = t_11 * qrs; { - auto _gst_258 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_258)); - *(reinterpret_cast(_gst_258) + (0)) = dg_o_11; + auto _gst_290 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_290)); + *(reinterpret_cast(_gst_290) + (0)) = dg_o_11; } if (t_11 < clen) { { - auto _gst_259 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_259)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_291 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_291)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_260 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_260)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_292 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_292)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3546,24 +3725,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; float q_r_12 = q_t0_12; - float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12)) + dgf_12; + float qe_11 = q_r_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11))) + dgf_12; int goff_12 = t_12 * rs_g; int qoff_12 = t_12 * qrs; { - auto _gst_261 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_261)); - *(reinterpret_cast(_gst_261) + (0)) = dg_o_12; + auto _gst_293 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_293)); + *(reinterpret_cast(_gst_293) + (0)) = dg_o_12; } if (t_12 < clen) { { - auto _gst_262 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_262)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_294 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_294)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_263 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_263)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_295 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_295)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3590,24 +3774,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; float q_r_13 = q_t0_13; - float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13)) + dgf_13; + float qe_12 = q_r_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12))) + dgf_13; int goff_13 = t_13 * rs_g; int qoff_13 = t_13 * qrs; { - auto _gst_264 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_264)); - *(reinterpret_cast(_gst_264) + (0)) = dg_o_13; + auto _gst_296 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_296)); + *(reinterpret_cast(_gst_296) + (0)) = dg_o_13; } if (t_13 < clen) { { - auto _gst_265 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_265)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_265) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_297 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_297)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_266 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_266)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_266) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_298 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_298)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3634,24 +3823,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; float q_r_14 = q_t0_14; - float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14)) + dgf_14; + float qe_13 = q_r_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13))) + dgf_14; int goff_14 = t_14 * rs_g; int qoff_14 = t_14 * qrs; { - auto _gst_267 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_o_14; + auto _gst_299 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_o_14; } if (t_14 < clen) { { - auto _gst_268 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_268) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_300 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_300) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_269 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_269) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_301 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3678,24 +3872,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; float q_r_15 = q_t0_15; - float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15)) + dgf_15; + float qe_14 = q_r_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14))) + dgf_15; int goff_15 = t_15 * rs_g; int qoff_15 = t_15 * qrs; { - auto _gst_270 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_o_15; + auto _gst_302 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_o_15; } if (t_15 < clen) { { - auto _gst_271 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_271) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_303 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_303) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_272 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_272) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_304 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dk_o_15; } } int b4_0 = lane_1 >> 4 & 1; @@ -3703,31 +3902,31 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_273 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_273; + float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_305; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_274 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_274; + float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_306; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_275 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_275; + float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_307; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_276 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_276; + float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_308; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_277 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_277; + float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_309; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_278 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_278; + float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_310; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_279 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_279; - float _shfl_xor_sync_280 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_280; - float _shfl_xor_sync_281 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_281; + float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_311; + float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_312; + float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_313; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); mbarrier_wait(p1b_ready_addr, 0); @@ -3787,24 +3986,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; float q_r_16 = q_t0_16; - float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16)) + dgf_16; + float qe_15 = q_r_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15))) + dgf_16; int goff_16 = t_16 * rs_g; int qoff_16 = t_16 * qrs; { - auto _gst_282 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_o_16; + auto _gst_314 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_o_16; } if (t_16 < clen) { { - auto _gst_283 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_283)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_315 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_315)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_315) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_284 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_316 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_17 = 16 + 8 * th + 1; @@ -3831,24 +4035,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; float q_r_17 = q_t0_17; - float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17)) + dgf_17; + float qe_16 = q_r_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16))) + dgf_17; int goff_17 = t_17 * rs_g; int qoff_17 = t_17 * qrs; { - auto _gst_285 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast(_gst_285) + (0)) = dg_o_17; + auto _gst_317 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast(_gst_317) + (0)) = dg_o_17; } if (t_17 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_18 = 16 + 8 * th + 2; @@ -3875,24 +4084,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; float q_r_18 = q_t0_18; - float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18)) + dgf_18; + float qe_17 = q_r_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17))) + dgf_18; int goff_18 = t_18 * rs_g; int qoff_18 = t_18 * qrs; { - auto _gst_288 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast(_gst_288) + (0)) = dg_o_18; + auto _gst_320 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast(_gst_320) + (0)) = dg_o_18; } if (t_18 < clen) { { - auto _gst_289 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_321 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_290 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_322 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_19 = 16 + 8 * th + 3; @@ -3919,24 +4133,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; float q_r_19 = q_t0_19; - float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19)) + dgf_19; + float qe_18 = q_r_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18))) + dgf_19; int goff_19 = t_19 * rs_g; int qoff_19 = t_19 * qrs; { - auto _gst_291 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast(_gst_291) + (0)) = dg_o_19; + auto _gst_323 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast(_gst_323) + (0)) = dg_o_19; } if (t_19 < clen) { { - auto _gst_292 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_324 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_293 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_325 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_20 = 16 + 8 * th + 4; @@ -3963,24 +4182,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; float q_r_20 = q_t0_20; - float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20)) + dgf_20; + float qe_19 = q_r_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19))) + dgf_20; int goff_20 = t_20 * rs_g; int qoff_20 = t_20 * qrs; { - auto _gst_294 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast(_gst_294) + (0)) = dg_o_20; + auto _gst_326 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast(_gst_326) + (0)) = dg_o_20; } if (t_20 < clen) { { - auto _gst_295 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_327 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_296 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_328 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_21 = 16 + 8 * th + 5; @@ -4007,24 +4231,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; float q_r_21 = q_t0_21; - float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21)) + dgf_21; + float qe_20 = q_r_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20))) + dgf_21; int goff_21 = t_21 * rs_g; int qoff_21 = t_21 * qrs; { - auto _gst_297 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast(_gst_297) + (0)) = dg_o_21; + auto _gst_329 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast(_gst_329) + (0)) = dg_o_21; } if (t_21 < clen) { { - auto _gst_298 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_330 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_299 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_331 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_22 = 16 + 8 * th + 6; @@ -4051,24 +4280,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; float q_r_22 = q_t0_22; - float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22)) + dgf_22; + float qe_21 = q_r_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21))) + dgf_22; int goff_22 = t_22 * rs_g; int qoff_22 = t_22 * qrs; { - auto _gst_300 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_300)); - *(reinterpret_cast(_gst_300) + (0)) = dg_o_22; + auto _gst_332 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_332)); + *(reinterpret_cast(_gst_332) + (0)) = dg_o_22; } if (t_22 < clen) { { - auto _gst_301 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_301)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_301) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_333 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_333)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_302 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_302)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_302) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_334 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_334)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_23 = 16 + 8 * th + 7; @@ -4095,24 +4329,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; float q_r_23 = q_t0_23; - float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23)) + dgf_23; + float qe_22 = q_r_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22))) + dgf_23; int goff_23 = t_23 * rs_g; int qoff_23 = t_23 * qrs; { - auto _gst_303 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_303)); - *(reinterpret_cast(_gst_303) + (0)) = dg_o_23; + auto _gst_335 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_335)); + *(reinterpret_cast(_gst_335) + (0)) = dg_o_23; } if (t_23 < clen) { { - auto _gst_304 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_304)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_304) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_336 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_336)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_336) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_305 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_305)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_305) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_337 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_337)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dk_o_23; } } int b4_1 = lane_1 >> 4 & 1; @@ -4120,31 +4359,31 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_306; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_338; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_307; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_339; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_308; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_340; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_309 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_309; + float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_341; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_310 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_310; + float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_342; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_311 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_311; + float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_343; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_312 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_312; - float _shfl_xor_sync_313 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_313; - float _shfl_xor_sync_314 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_314; + float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_344; + float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_345; + float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_346; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); mbarrier_wait(epi_full_addr + (th) * 8, 0); @@ -4198,24 +4437,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; float q_r_24 = q_t0_24; - float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24)) + dgf_24; + float qe_23 = q_r_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23))) + dgf_24; int goff_24 = t_24 * rs_g; int qoff_24 = t_24 * qrs; { - auto _gst_315 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_o_24; + auto _gst_347 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_o_24; } if (t_24 < clen) { { - auto _gst_316 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_348 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_348) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_317 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_349 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_349) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_25 = 8 * th + 1; @@ -4242,24 +4486,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; float q_r_25 = q_t0_25; - float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25)) + dgf_25; + float qe_24 = q_r_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24))) + dgf_25; int goff_25 = t_25 * rs_g; int qoff_25 = t_25 * qrs; { - auto _gst_318 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_318)); - *(reinterpret_cast(_gst_318) + (0)) = dg_o_25; + auto _gst_350 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_350)); + *(reinterpret_cast(_gst_350) + (0)) = dg_o_25; } if (t_25 < clen) { { - auto _gst_319 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_351 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_320 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_352 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_26 = 8 * th + 2; @@ -4286,24 +4535,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; float q_r_26 = q_t0_26; - float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26)) + dgf_26; + float qe_25 = q_r_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25))) + dgf_26; int goff_26 = t_26 * rs_g; int qoff_26 = t_26 * qrs; { - auto _gst_321 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast(_gst_321) + (0)) = dg_o_26; + auto _gst_353 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast(_gst_353) + (0)) = dg_o_26; } if (t_26 < clen) { { - auto _gst_322 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_354 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_323 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_355 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_27 = 8 * th + 3; @@ -4330,24 +4584,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; float q_r_27 = q_t0_27; - float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27)) + dgf_27; + float qe_26 = q_r_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26))) + dgf_27; int goff_27 = t_27 * rs_g; int qoff_27 = t_27 * qrs; { - auto _gst_324 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast(_gst_324) + (0)) = dg_o_27; + auto _gst_356 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast(_gst_356) + (0)) = dg_o_27; } if (t_27 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_28 = 8 * th + 4; @@ -4374,24 +4633,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; float q_r_28 = q_t0_28; - float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28)) + dgf_28; + float qe_27 = q_r_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27))) + dgf_28; int goff_28 = t_28 * rs_g; int qoff_28 = t_28 * qrs; { - auto _gst_327 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast(_gst_327) + (0)) = dg_o_28; + auto _gst_359 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast(_gst_359) + (0)) = dg_o_28; } if (t_28 < clen) { { - auto _gst_328 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_360 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_329 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_361 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_29 = 8 * th + 5; @@ -4418,24 +4682,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; float q_r_29 = q_t0_29; - float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29)) + dgf_29; + float qe_28 = q_r_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28))) + dgf_29; int goff_29 = t_29 * rs_g; int qoff_29 = t_29 * qrs; { - auto _gst_330 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast(_gst_330) + (0)) = dg_o_29; + auto _gst_362 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast(_gst_362) + (0)) = dg_o_29; } if (t_29 < clen) { { - auto _gst_331 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_363 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_332 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_364 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_30 = 8 * th + 6; @@ -4462,24 +4731,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; float q_r_30 = q_t0_30; - float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30)) + dgf_30; + float qe_29 = q_r_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29))) + dgf_30; int goff_30 = t_30 * rs_g; int qoff_30 = t_30 * qrs; { - auto _gst_333 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast(_gst_333) + (0)) = dg_o_30; + auto _gst_365 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast(_gst_365) + (0)) = dg_o_30; } if (t_30 < clen) { { - auto _gst_334 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_366 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_335 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_335)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_335) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_367 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_367)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_367) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_31 = 8 * th + 7; @@ -4506,24 +4780,29 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; float q_r_31 = q_t0_31; - float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31)) + dgf_31; + float qe_30 = q_r_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30))) + dgf_31; int goff_31 = t_31 * rs_g; int qoff_31 = t_31 * qrs; { - auto _gst_336 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_336)); - *(reinterpret_cast(_gst_336) + (0)) = dg_o_31; + auto _gst_368 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_368)); + *(reinterpret_cast(_gst_368) + (0)) = dg_o_31; } if (t_31 < clen) { { - auto _gst_337 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_337)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_337) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_369 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_369)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_369) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_338 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_338)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_338) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_370 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_370)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_370) + (0)) = (__nv_bfloat16)dk_o_31; } } int b4_2 = lane_1 >> 4 & 1; @@ -4531,31 +4810,31 @@ kernel_cake_kda_train_d5fd3f7234a4266e09d2(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_371; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_372; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_341; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_373; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_342; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_374; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_343; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_375; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_344 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_344; + float _shfl_xor_sync_376 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_376; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_345 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_345; - float _shfl_xor_sync_346 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_346; - float _shfl_xor_sync_347 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_347; + float _shfl_xor_sync_377 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_377; + float _shfl_xor_sync_378 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_378; + float _shfl_xor_sync_379 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_379; if ((lane_1 & 3) == 0) { int e_star = lane_1 >> 2 & 7; pool_f32[28160 + qd * 64 + 8 * th + e_star] = v1_2; diff --git a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu index 26cc6d1aab8..51e16f3e7b4 100644 --- a/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu +++ b/flashinfer/experimental/cake/cake_kda_train/csrc/cake_kda_train_f4ea302627f05d805437_kernel.cu @@ -139,6 +139,13 @@ __device__ __forceinline__ float approx_exp2(float x) { } +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + __device__ __forceinline__ float2 sub_f32x2(float2 a, float2 b) { float2 r; asm("sub.rn.ftz.f32x2 %0, %1, %2;" @@ -181,6 +188,22 @@ __device__ __forceinline__ void tmem_ld_x8(float* dst, int tmem_addr) { : "r"(tmem_addr)); } + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void @@ -2246,15 +2269,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_0 = sub_f32x2(gc2, g2_0); float e2_0_0 = approx_exp2(dc2_0.x); float e2_1_0 = approx_exp2(dc2_0.y); - float2 dn2_0 = sub_f32x2(gnl2, g2_0); - float en_0_0 = approx_exp2(dn2_0.x); - float en_1_0 = approx_exp2(dn2_0.y); - dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * e2_0_0); - dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * e2_1_0); float2 _f32x2_mul_a_168 = make_float2(k_t0_0, k_t1_0); float2 _f32x2_mul_b_169 = make_float2(e2_0_0, e2_1_0); float2 p2_0; asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_168), "l"(*(const unsigned long long*)&_f32x2_mul_b_169)); + float dl = __uint_as_float(__as_u32(p2_0.x) & 4294959104u) - p2_0.x; + float _rcp_170 = approx_rcp(k_t0_0); + float ft = ((dl == 0.0f) ? e2_0_0 : __fmaf_rn(dl, _rcp_170, e2_0_0)); + float dl_0 = __uint_as_float(__as_u32(p2_0.y) & 4294959104u) - p2_0.y; + float _rcp_171 = approx_rcp(k_t1_0); + float ft_0 = ((dl_0 == 0.0f) ? e2_1_0 : __fmaf_rn(dl_0, _rcp_171, e2_1_0)); + float2 dn2_0 = sub_f32x2(gnl2, g2_0); + float en_0_0 = approx_exp2(dn2_0.x); + float en_1_0 = approx_exp2(dn2_0.y); + dkt[0] = __fmaf_rn(offv[0], en_0_0, dgv[0] * ft); + dkt[1] = __fmaf_rn(offv[1], en_1_0, dgv[1] * ft_0); pan[0] = p2_0.x; pan[1] = p2_0.y; int t0_1 = 32 + 8 * th + 2; @@ -2275,15 +2304,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_1 = sub_f32x2(gc2, g2_1); float e2_0_1 = approx_exp2(dc2_1.x); float e2_1_1 = approx_exp2(dc2_1.y); + float2 _f32x2_mul_a_172 = make_float2(k_t0_1, k_t1_1); + float2 _f32x2_mul_b_173 = make_float2(e2_0_1, e2_1_1); + float2 p2_1; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + float dl_1 = __uint_as_float(__as_u32(p2_1.x) & 4294959104u) - p2_1.x; + float _rcp_174 = approx_rcp(k_t0_1); + float ft_1 = ((dl_1 == 0.0f) ? e2_0_1 : __fmaf_rn(dl_1, _rcp_174, e2_0_1)); + float dl_2 = __uint_as_float(__as_u32(p2_1.y) & 4294959104u) - p2_1.y; + float _rcp_175 = approx_rcp(k_t1_1); + float ft_2 = ((dl_2 == 0.0f) ? e2_1_1 : __fmaf_rn(dl_2, _rcp_175, e2_1_1)); float2 dn2_1 = sub_f32x2(gnl2, g2_1); float en_0_1 = approx_exp2(dn2_1.x); float en_1_1 = approx_exp2(dn2_1.y); - dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * e2_0_1); - dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * e2_1_1); - float2 _f32x2_mul_a_170 = make_float2(k_t0_1, k_t1_1); - float2 _f32x2_mul_b_171 = make_float2(e2_0_1, e2_1_1); - float2 p2_1; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_170), "l"(*(const unsigned long long*)&_f32x2_mul_b_171)); + dkt[2] = __fmaf_rn(offv[2], en_0_1, dgv[2] * ft_1); + dkt[3] = __fmaf_rn(offv[3], en_1_1, dgv[3] * ft_2); pan[2] = p2_1.x; pan[3] = p2_1.y; int t0_2 = 32 + 8 * th + 4; @@ -2304,15 +2339,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_2 = sub_f32x2(gc2, g2_2); float e2_0_2 = approx_exp2(dc2_2.x); float e2_1_2 = approx_exp2(dc2_2.y); + float2 _f32x2_mul_a_176 = make_float2(k_t0_2, k_t1_2); + float2 _f32x2_mul_b_177 = make_float2(e2_0_2, e2_1_2); + float2 p2_2; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + float dl_3 = __uint_as_float(__as_u32(p2_2.x) & 4294959104u) - p2_2.x; + float _rcp_178 = approx_rcp(k_t0_2); + float ft_3 = ((dl_3 == 0.0f) ? e2_0_2 : __fmaf_rn(dl_3, _rcp_178, e2_0_2)); + float dl_4 = __uint_as_float(__as_u32(p2_2.y) & 4294959104u) - p2_2.y; + float _rcp_179 = approx_rcp(k_t1_2); + float ft_4 = ((dl_4 == 0.0f) ? e2_1_2 : __fmaf_rn(dl_4, _rcp_179, e2_1_2)); float2 dn2_2 = sub_f32x2(gnl2, g2_2); float en_0_2 = approx_exp2(dn2_2.x); float en_1_2 = approx_exp2(dn2_2.y); - dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * e2_0_2); - dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * e2_1_2); - float2 _f32x2_mul_a_172 = make_float2(k_t0_2, k_t1_2); - float2 _f32x2_mul_b_173 = make_float2(e2_0_2, e2_1_2); - float2 p2_2; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_172), "l"(*(const unsigned long long*)&_f32x2_mul_b_173)); + dkt[4] = __fmaf_rn(offv[4], en_0_2, dgv[4] * ft_3); + dkt[5] = __fmaf_rn(offv[5], en_1_2, dgv[5] * ft_4); pan[4] = p2_2.x; pan[5] = p2_2.y; int t0_3 = 32 + 8 * th + 6; @@ -2333,15 +2374,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_3 = sub_f32x2(gc2, g2_3); float e2_0_3 = approx_exp2(dc2_3.x); float e2_1_3 = approx_exp2(dc2_3.y); + float2 _f32x2_mul_a_180 = make_float2(k_t0_3, k_t1_3); + float2 _f32x2_mul_b_181 = make_float2(e2_0_3, e2_1_3); + float2 p2_3; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + float dl_5 = __uint_as_float(__as_u32(p2_3.x) & 4294959104u) - p2_3.x; + float _rcp_182 = approx_rcp(k_t0_3); + float ft_5 = ((dl_5 == 0.0f) ? e2_0_3 : __fmaf_rn(dl_5, _rcp_182, e2_0_3)); + float dl_6 = __uint_as_float(__as_u32(p2_3.y) & 4294959104u) - p2_3.y; + float _rcp_183 = approx_rcp(k_t1_3); + float ft_6 = ((dl_6 == 0.0f) ? e2_1_3 : __fmaf_rn(dl_6, _rcp_183, e2_1_3)); float2 dn2_3 = sub_f32x2(gnl2, g2_3); float en_0_3 = approx_exp2(dn2_3.x); float en_1_3 = approx_exp2(dn2_3.y); - dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * e2_0_3); - dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * e2_1_3); - float2 _f32x2_mul_a_174 = make_float2(k_t0_3, k_t1_3); - float2 _f32x2_mul_b_175 = make_float2(e2_0_3, e2_1_3); - float2 p2_3; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_174), "l"(*(const unsigned long long*)&_f32x2_mul_b_175)); + dkt[6] = __fmaf_rn(offv[6], en_0_3, dgv[6] * ft_5); + dkt[7] = __fmaf_rn(offv[7], en_1_3, dgv[7] * ft_6); pan[6] = p2_3.x; pan[7] = p2_3.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(32 + 8 * th), dkt); @@ -2377,12 +2424,18 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_4 = sub_f32x2(gc2_0, make_float2(g_t0_4, g_t1_4)); float e2_0_4 = approx_exp2(dc2_4.x); float e2_1_4 = approx_exp2(dc2_4.y); - dkt_0[0] = dgv_0[0] * e2_0_4; - dkt_0[1] = dgv_0[1] * e2_1_4; - float2 _f32x2_mul_a_176 = make_float2(k_t0_4, k_t1_4); - float2 _f32x2_mul_b_177 = make_float2(e2_0_4, e2_1_4); + float2 _f32x2_mul_a_184 = make_float2(k_t0_4, k_t1_4); + float2 _f32x2_mul_b_185 = make_float2(e2_0_4, e2_1_4); float2 p2_4; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_176), "l"(*(const unsigned long long*)&_f32x2_mul_b_177)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + float dl_7 = __uint_as_float(__as_u32(p2_4.x) & 4294959104u) - p2_4.x; + float _rcp_186 = approx_rcp(k_t0_4); + float ft_7 = ((dl_7 == 0.0f) ? e2_0_4 : __fmaf_rn(dl_7, _rcp_186, e2_0_4)); + float dl_8 = __uint_as_float(__as_u32(p2_4.y) & 4294959104u) - p2_4.y; + float _rcp_187 = approx_rcp(k_t1_4); + float ft_8 = ((dl_8 == 0.0f) ? e2_1_4 : __fmaf_rn(dl_8, _rcp_187, e2_1_4)); + dkt_0[0] = dgv_0[0] * ft_7; + dkt_0[1] = dgv_0[1] * ft_8; pan_0[0] = p2_4.x; pan_0[1] = p2_4.y; int t0_5 = 48 + 8 * th + 2; @@ -2402,12 +2455,18 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_5 = sub_f32x2(gc2_0, make_float2(g_t0_5, g_t1_5)); float e2_0_5 = approx_exp2(dc2_5.x); float e2_1_5 = approx_exp2(dc2_5.y); - dkt_0[2] = dgv_0[2] * e2_0_5; - dkt_0[3] = dgv_0[3] * e2_1_5; - float2 _f32x2_mul_a_178 = make_float2(k_t0_5, k_t1_5); - float2 _f32x2_mul_b_179 = make_float2(e2_0_5, e2_1_5); + float2 _f32x2_mul_a_188 = make_float2(k_t0_5, k_t1_5); + float2 _f32x2_mul_b_189 = make_float2(e2_0_5, e2_1_5); float2 p2_5; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_178), "l"(*(const unsigned long long*)&_f32x2_mul_b_179)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + float dl_9 = __uint_as_float(__as_u32(p2_5.x) & 4294959104u) - p2_5.x; + float _rcp_190 = approx_rcp(k_t0_5); + float ft_9 = ((dl_9 == 0.0f) ? e2_0_5 : __fmaf_rn(dl_9, _rcp_190, e2_0_5)); + float dl_10 = __uint_as_float(__as_u32(p2_5.y) & 4294959104u) - p2_5.y; + float _rcp_191 = approx_rcp(k_t1_5); + float ft_10 = ((dl_10 == 0.0f) ? e2_1_5 : __fmaf_rn(dl_10, _rcp_191, e2_1_5)); + dkt_0[2] = dgv_0[2] * ft_9; + dkt_0[3] = dgv_0[3] * ft_10; pan_0[2] = p2_5.x; pan_0[3] = p2_5.y; int t0_6 = 48 + 8 * th + 4; @@ -2427,12 +2486,18 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_6 = sub_f32x2(gc2_0, make_float2(g_t0_6, g_t1_6)); float e2_0_6 = approx_exp2(dc2_6.x); float e2_1_6 = approx_exp2(dc2_6.y); - dkt_0[4] = dgv_0[4] * e2_0_6; - dkt_0[5] = dgv_0[5] * e2_1_6; - float2 _f32x2_mul_a_180 = make_float2(k_t0_6, k_t1_6); - float2 _f32x2_mul_b_181 = make_float2(e2_0_6, e2_1_6); + float2 _f32x2_mul_a_192 = make_float2(k_t0_6, k_t1_6); + float2 _f32x2_mul_b_193 = make_float2(e2_0_6, e2_1_6); float2 p2_6; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_180), "l"(*(const unsigned long long*)&_f32x2_mul_b_181)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + float dl_11 = __uint_as_float(__as_u32(p2_6.x) & 4294959104u) - p2_6.x; + float _rcp_194 = approx_rcp(k_t0_6); + float ft_11 = ((dl_11 == 0.0f) ? e2_0_6 : __fmaf_rn(dl_11, _rcp_194, e2_0_6)); + float dl_12 = __uint_as_float(__as_u32(p2_6.y) & 4294959104u) - p2_6.y; + float _rcp_195 = approx_rcp(k_t1_6); + float ft_12 = ((dl_12 == 0.0f) ? e2_1_6 : __fmaf_rn(dl_12, _rcp_195, e2_1_6)); + dkt_0[4] = dgv_0[4] * ft_11; + dkt_0[5] = dgv_0[5] * ft_12; pan_0[4] = p2_6.x; pan_0[5] = p2_6.y; int t0_7 = 48 + 8 * th + 6; @@ -2452,12 +2517,18 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_7 = sub_f32x2(gc2_0, make_float2(g_t0_7, g_t1_7)); float e2_0_7 = approx_exp2(dc2_7.x); float e2_1_7 = approx_exp2(dc2_7.y); - dkt_0[6] = dgv_0[6] * e2_0_7; - dkt_0[7] = dgv_0[7] * e2_1_7; - float2 _f32x2_mul_a_182 = make_float2(k_t0_7, k_t1_7); - float2 _f32x2_mul_b_183 = make_float2(e2_0_7, e2_1_7); + float2 _f32x2_mul_a_196 = make_float2(k_t0_7, k_t1_7); + float2 _f32x2_mul_b_197 = make_float2(e2_0_7, e2_1_7); float2 p2_7; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_182), "l"(*(const unsigned long long*)&_f32x2_mul_b_183)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + float dl_13 = __uint_as_float(__as_u32(p2_7.x) & 4294959104u) - p2_7.x; + float _rcp_198 = approx_rcp(k_t0_7); + float ft_13 = ((dl_13 == 0.0f) ? e2_0_7 : __fmaf_rn(dl_13, _rcp_198, e2_0_7)); + float dl_14 = __uint_as_float(__as_u32(p2_7.y) & 4294959104u) - p2_7.y; + float _rcp_199 = approx_rcp(k_t1_7); + float ft_14 = ((dl_14 == 0.0f) ? e2_1_7 : __fmaf_rn(dl_14, _rcp_199, e2_1_7)); + dkt_0[6] = dgv_0[6] * ft_13; + dkt_0[7] = dgv_0[7] * ft_14; pan_0[6] = p2_7.x; pan_0[7] = p2_7.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(48 + 8 * th), dkt_0); @@ -2502,15 +2573,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_8 = sub_f32x2(gc2_1, g2_4); float e2_0_8 = approx_exp2(dc2_8.x); float e2_1_8 = approx_exp2(dc2_8.y); + float2 _f32x2_mul_a_200 = make_float2(k_t0_8, k_t1_8); + float2 _f32x2_mul_b_201 = make_float2(e2_0_8, e2_1_8); + float2 p2_8; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + float dl_15 = __uint_as_float(__as_u32(p2_8.x) & 4294959104u) - p2_8.x; + float _rcp_202 = approx_rcp(k_t0_8); + float ft_15 = ((dl_15 == 0.0f) ? e2_0_8 : __fmaf_rn(dl_15, _rcp_202, e2_0_8)); + float dl_16 = __uint_as_float(__as_u32(p2_8.y) & 4294959104u) - p2_8.y; + float _rcp_203 = approx_rcp(k_t1_8); + float ft_16 = ((dl_16 == 0.0f) ? e2_1_8 : __fmaf_rn(dl_16, _rcp_203, e2_1_8)); float2 dn2_4 = sub_f32x2(gnl2_1, g2_4); float en_0_4 = approx_exp2(dn2_4.x); float en_1_4 = approx_exp2(dn2_4.y); - dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * e2_0_8); - dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * e2_1_8); - float2 _f32x2_mul_a_184 = make_float2(k_t0_8, k_t1_8); - float2 _f32x2_mul_b_185 = make_float2(e2_0_8, e2_1_8); - float2 p2_8; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_184), "l"(*(const unsigned long long*)&_f32x2_mul_b_185)); + dkt_1[0] = __fmaf_rn(offv_0[0], en_0_4, dgv_1[0] * ft_15); + dkt_1[1] = __fmaf_rn(offv_0[1], en_1_4, dgv_1[1] * ft_16); pan_1[0] = p2_8.x; pan_1[1] = p2_8.y; int t0_9 = 8 * th + 2; @@ -2531,15 +2608,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_9 = sub_f32x2(gc2_1, g2_5); float e2_0_9 = approx_exp2(dc2_9.x); float e2_1_9 = approx_exp2(dc2_9.y); + float2 _f32x2_mul_a_204 = make_float2(k_t0_9, k_t1_9); + float2 _f32x2_mul_b_205 = make_float2(e2_0_9, e2_1_9); + float2 p2_9; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + float dl_17 = __uint_as_float(__as_u32(p2_9.x) & 4294959104u) - p2_9.x; + float _rcp_206 = approx_rcp(k_t0_9); + float ft_17 = ((dl_17 == 0.0f) ? e2_0_9 : __fmaf_rn(dl_17, _rcp_206, e2_0_9)); + float dl_18 = __uint_as_float(__as_u32(p2_9.y) & 4294959104u) - p2_9.y; + float _rcp_207 = approx_rcp(k_t1_9); + float ft_18 = ((dl_18 == 0.0f) ? e2_1_9 : __fmaf_rn(dl_18, _rcp_207, e2_1_9)); float2 dn2_5 = sub_f32x2(gnl2_1, g2_5); float en_0_5 = approx_exp2(dn2_5.x); float en_1_5 = approx_exp2(dn2_5.y); - dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * e2_0_9); - dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * e2_1_9); - float2 _f32x2_mul_a_186 = make_float2(k_t0_9, k_t1_9); - float2 _f32x2_mul_b_187 = make_float2(e2_0_9, e2_1_9); - float2 p2_9; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_186), "l"(*(const unsigned long long*)&_f32x2_mul_b_187)); + dkt_1[2] = __fmaf_rn(offv_0[2], en_0_5, dgv_1[2] * ft_17); + dkt_1[3] = __fmaf_rn(offv_0[3], en_1_5, dgv_1[3] * ft_18); pan_1[2] = p2_9.x; pan_1[3] = p2_9.y; int t0_10 = 8 * th + 4; @@ -2560,15 +2643,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_10 = sub_f32x2(gc2_1, g2_6); float e2_0_10 = approx_exp2(dc2_10.x); float e2_1_10 = approx_exp2(dc2_10.y); + float2 _f32x2_mul_a_208 = make_float2(k_t0_10, k_t1_10); + float2 _f32x2_mul_b_209 = make_float2(e2_0_10, e2_1_10); + float2 p2_10; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_208), "l"(*(const unsigned long long*)&_f32x2_mul_b_209)); + float dl_19 = __uint_as_float(__as_u32(p2_10.x) & 4294959104u) - p2_10.x; + float _rcp_210 = approx_rcp(k_t0_10); + float ft_19 = ((dl_19 == 0.0f) ? e2_0_10 : __fmaf_rn(dl_19, _rcp_210, e2_0_10)); + float dl_20 = __uint_as_float(__as_u32(p2_10.y) & 4294959104u) - p2_10.y; + float _rcp_211 = approx_rcp(k_t1_10); + float ft_20 = ((dl_20 == 0.0f) ? e2_1_10 : __fmaf_rn(dl_20, _rcp_211, e2_1_10)); float2 dn2_6 = sub_f32x2(gnl2_1, g2_6); float en_0_6 = approx_exp2(dn2_6.x); float en_1_6 = approx_exp2(dn2_6.y); - dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * e2_0_10); - dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * e2_1_10); - float2 _f32x2_mul_a_188 = make_float2(k_t0_10, k_t1_10); - float2 _f32x2_mul_b_189 = make_float2(e2_0_10, e2_1_10); - float2 p2_10; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_188), "l"(*(const unsigned long long*)&_f32x2_mul_b_189)); + dkt_1[4] = __fmaf_rn(offv_0[4], en_0_6, dgv_1[4] * ft_19); + dkt_1[5] = __fmaf_rn(offv_0[5], en_1_6, dgv_1[5] * ft_20); pan_1[4] = p2_10.x; pan_1[5] = p2_10.y; int t0_11 = 8 * th + 6; @@ -2589,15 +2678,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_11 = sub_f32x2(gc2_1, g2_7); float e2_0_11 = approx_exp2(dc2_11.x); float e2_1_11 = approx_exp2(dc2_11.y); + float2 _f32x2_mul_a_212 = make_float2(k_t0_11, k_t1_11); + float2 _f32x2_mul_b_213 = make_float2(e2_0_11, e2_1_11); + float2 p2_11; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_212), "l"(*(const unsigned long long*)&_f32x2_mul_b_213)); + float dl_21 = __uint_as_float(__as_u32(p2_11.x) & 4294959104u) - p2_11.x; + float _rcp_214 = approx_rcp(k_t0_11); + float ft_21 = ((dl_21 == 0.0f) ? e2_0_11 : __fmaf_rn(dl_21, _rcp_214, e2_0_11)); + float dl_22 = __uint_as_float(__as_u32(p2_11.y) & 4294959104u) - p2_11.y; + float _rcp_215 = approx_rcp(k_t1_11); + float ft_22 = ((dl_22 == 0.0f) ? e2_1_11 : __fmaf_rn(dl_22, _rcp_215, e2_1_11)); float2 dn2_7 = sub_f32x2(gnl2_1, g2_7); float en_0_7 = approx_exp2(dn2_7.x); float en_1_7 = approx_exp2(dn2_7.y); - dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * e2_0_11); - dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * e2_1_11); - float2 _f32x2_mul_a_190 = make_float2(k_t0_11, k_t1_11); - float2 _f32x2_mul_b_191 = make_float2(e2_0_11, e2_1_11); - float2 p2_11; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_190), "l"(*(const unsigned long long*)&_f32x2_mul_b_191)); + dkt_1[6] = __fmaf_rn(offv_0[6], en_0_7, dgv_1[6] * ft_21); + dkt_1[7] = __fmaf_rn(offv_0[7], en_1_7, dgv_1[7] * ft_22); pan_1[6] = p2_11.x; pan_1[7] = p2_11.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(8 * th), dkt_1); @@ -2636,15 +2731,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_12 = sub_f32x2(gc2_2, g2_8); float e2_0_12 = approx_exp2(dc2_12.x); float e2_1_12 = approx_exp2(dc2_12.y); + float2 _f32x2_mul_a_216 = make_float2(k_t0_12, k_t1_12); + float2 _f32x2_mul_b_217 = make_float2(e2_0_12, e2_1_12); + float2 p2_12; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_216), "l"(*(const unsigned long long*)&_f32x2_mul_b_217)); + float dl_23 = __uint_as_float(__as_u32(p2_12.x) & 4294959104u) - p2_12.x; + float _rcp_218 = approx_rcp(k_t0_12); + float ft_23 = ((dl_23 == 0.0f) ? e2_0_12 : __fmaf_rn(dl_23, _rcp_218, e2_0_12)); + float dl_24 = __uint_as_float(__as_u32(p2_12.y) & 4294959104u) - p2_12.y; + float _rcp_219 = approx_rcp(k_t1_12); + float ft_24 = ((dl_24 == 0.0f) ? e2_1_12 : __fmaf_rn(dl_24, _rcp_219, e2_1_12)); float2 dn2_8 = sub_f32x2(gnl2_2, g2_8); float en_0_8 = approx_exp2(dn2_8.x); float en_1_8 = approx_exp2(dn2_8.y); - dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * e2_0_12); - dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * e2_1_12); - float2 _f32x2_mul_a_192 = make_float2(k_t0_12, k_t1_12); - float2 _f32x2_mul_b_193 = make_float2(e2_0_12, e2_1_12); - float2 p2_12; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_192), "l"(*(const unsigned long long*)&_f32x2_mul_b_193)); + dkt_2[0] = __fmaf_rn(offv_1[0], en_0_8, dgv_2[0] * ft_23); + dkt_2[1] = __fmaf_rn(offv_1[1], en_1_8, dgv_2[1] * ft_24); pan_2[0] = p2_12.x; pan_2[1] = p2_12.y; int t0_13 = 16 + 8 * th + 2; @@ -2665,15 +2766,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_13 = sub_f32x2(gc2_2, g2_9); float e2_0_13 = approx_exp2(dc2_13.x); float e2_1_13 = approx_exp2(dc2_13.y); + float2 _f32x2_mul_a_220 = make_float2(k_t0_13, k_t1_13); + float2 _f32x2_mul_b_221 = make_float2(e2_0_13, e2_1_13); + float2 p2_13; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_220), "l"(*(const unsigned long long*)&_f32x2_mul_b_221)); + float dl_25 = __uint_as_float(__as_u32(p2_13.x) & 4294959104u) - p2_13.x; + float _rcp_222 = approx_rcp(k_t0_13); + float ft_25 = ((dl_25 == 0.0f) ? e2_0_13 : __fmaf_rn(dl_25, _rcp_222, e2_0_13)); + float dl_26 = __uint_as_float(__as_u32(p2_13.y) & 4294959104u) - p2_13.y; + float _rcp_223 = approx_rcp(k_t1_13); + float ft_26 = ((dl_26 == 0.0f) ? e2_1_13 : __fmaf_rn(dl_26, _rcp_223, e2_1_13)); float2 dn2_9 = sub_f32x2(gnl2_2, g2_9); float en_0_9 = approx_exp2(dn2_9.x); float en_1_9 = approx_exp2(dn2_9.y); - dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * e2_0_13); - dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * e2_1_13); - float2 _f32x2_mul_a_194 = make_float2(k_t0_13, k_t1_13); - float2 _f32x2_mul_b_195 = make_float2(e2_0_13, e2_1_13); - float2 p2_13; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_194), "l"(*(const unsigned long long*)&_f32x2_mul_b_195)); + dkt_2[2] = __fmaf_rn(offv_1[2], en_0_9, dgv_2[2] * ft_25); + dkt_2[3] = __fmaf_rn(offv_1[3], en_1_9, dgv_2[3] * ft_26); pan_2[2] = p2_13.x; pan_2[3] = p2_13.y; int t0_14 = 16 + 8 * th + 4; @@ -2694,15 +2801,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_14 = sub_f32x2(gc2_2, g2_10); float e2_0_14 = approx_exp2(dc2_14.x); float e2_1_14 = approx_exp2(dc2_14.y); + float2 _f32x2_mul_a_224 = make_float2(k_t0_14, k_t1_14); + float2 _f32x2_mul_b_225 = make_float2(e2_0_14, e2_1_14); + float2 p2_14; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_224), "l"(*(const unsigned long long*)&_f32x2_mul_b_225)); + float dl_27 = __uint_as_float(__as_u32(p2_14.x) & 4294959104u) - p2_14.x; + float _rcp_226 = approx_rcp(k_t0_14); + float ft_27 = ((dl_27 == 0.0f) ? e2_0_14 : __fmaf_rn(dl_27, _rcp_226, e2_0_14)); + float dl_28 = __uint_as_float(__as_u32(p2_14.y) & 4294959104u) - p2_14.y; + float _rcp_227 = approx_rcp(k_t1_14); + float ft_28 = ((dl_28 == 0.0f) ? e2_1_14 : __fmaf_rn(dl_28, _rcp_227, e2_1_14)); float2 dn2_10 = sub_f32x2(gnl2_2, g2_10); float en_0_10 = approx_exp2(dn2_10.x); float en_1_10 = approx_exp2(dn2_10.y); - dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * e2_0_14); - dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * e2_1_14); - float2 _f32x2_mul_a_196 = make_float2(k_t0_14, k_t1_14); - float2 _f32x2_mul_b_197 = make_float2(e2_0_14, e2_1_14); - float2 p2_14; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_196), "l"(*(const unsigned long long*)&_f32x2_mul_b_197)); + dkt_2[4] = __fmaf_rn(offv_1[4], en_0_10, dgv_2[4] * ft_27); + dkt_2[5] = __fmaf_rn(offv_1[5], en_1_10, dgv_2[5] * ft_28); pan_2[4] = p2_14.x; pan_2[5] = p2_14.y; int t0_15 = 16 + 8 * th + 6; @@ -2723,15 +2836,21 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 dc2_15 = sub_f32x2(gc2_2, g2_11); float e2_0_15 = approx_exp2(dc2_15.x); float e2_1_15 = approx_exp2(dc2_15.y); + float2 _f32x2_mul_a_228 = make_float2(k_t0_15, k_t1_15); + float2 _f32x2_mul_b_229 = make_float2(e2_0_15, e2_1_15); + float2 p2_15; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_228), "l"(*(const unsigned long long*)&_f32x2_mul_b_229)); + float dl_29 = __uint_as_float(__as_u32(p2_15.x) & 4294959104u) - p2_15.x; + float _rcp_230 = approx_rcp(k_t0_15); + float ft_29 = ((dl_29 == 0.0f) ? e2_0_15 : __fmaf_rn(dl_29, _rcp_230, e2_0_15)); + float dl_30 = __uint_as_float(__as_u32(p2_15.y) & 4294959104u) - p2_15.y; + float _rcp_231 = approx_rcp(k_t1_15); + float ft_30 = ((dl_30 == 0.0f) ? e2_1_15 : __fmaf_rn(dl_30, _rcp_231, e2_1_15)); float2 dn2_11 = sub_f32x2(gnl2_2, g2_11); float en_0_11 = approx_exp2(dn2_11.x); float en_1_11 = approx_exp2(dn2_11.y); - dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * e2_0_15); - dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * e2_1_15); - float2 _f32x2_mul_a_198 = make_float2(k_t0_15, k_t1_15); - float2 _f32x2_mul_b_199 = make_float2(e2_0_15, e2_1_15); - float2 p2_15; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&p2_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_198), "l"(*(const unsigned long long*)&_f32x2_mul_b_199)); + dkt_2[6] = __fmaf_rn(offv_1[6], en_0_11, dgv_2[6] * ft_29); + dkt_2[7] = __fmaf_rn(offv_1[7], en_1_11, dgv_2[7] * ft_30); pan_2[6] = p2_15.x; pan_2[7] = p2_15.y; tmem_st_x8_f32(taddr + (unsigned int)row_base + (unsigned int)(16 + 8 * th), dkt_2); @@ -2802,37 +2921,37 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float2 d2_60 = sub_f32x2(a2_14, make_float2(gs_2[0], gs_2[1])); float e0_20 = approx_exp2(d2_60.x); float e1_20 = approx_exp2(d2_60.y); - float2 _f32x2_mul_a_200 = make_float2(ks_2[0], ks_2[1]); - float2 _f32x2_mul_b_201 = make_float2(e0_20, e1_20); + float2 _f32x2_mul_a_232 = make_float2(ks_2[0], ks_2[1]); + float2 _f32x2_mul_b_233 = make_float2(e0_20, e1_20); float2 v2_20; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_200), "l"(*(const unsigned long long*)&_f32x2_mul_b_201)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_232), "l"(*(const unsigned long long*)&_f32x2_mul_b_233)); v_5[0] = v2_20.x; v_5[1] = v2_20.y; float2 d2_61 = sub_f32x2(a2_14, make_float2(gs_2[2], gs_2[3])); float e0_21 = approx_exp2(d2_61.x); float e1_21 = approx_exp2(d2_61.y); - float2 _f32x2_mul_a_202 = make_float2(ks_2[2], ks_2[3]); - float2 _f32x2_mul_b_203 = make_float2(e0_21, e1_21); + float2 _f32x2_mul_a_234 = make_float2(ks_2[2], ks_2[3]); + float2 _f32x2_mul_b_235 = make_float2(e0_21, e1_21); float2 v2_21; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_202), "l"(*(const unsigned long long*)&_f32x2_mul_b_203)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_234), "l"(*(const unsigned long long*)&_f32x2_mul_b_235)); v_5[2] = v2_21.x; v_5[3] = v2_21.y; float2 d2_62 = sub_f32x2(a2_14, make_float2(gs_2[4], gs_2[5])); float e0_22 = approx_exp2(d2_62.x); float e1_22 = approx_exp2(d2_62.y); - float2 _f32x2_mul_a_204 = make_float2(ks_2[4], ks_2[5]); - float2 _f32x2_mul_b_205 = make_float2(e0_22, e1_22); + float2 _f32x2_mul_a_236 = make_float2(ks_2[4], ks_2[5]); + float2 _f32x2_mul_b_237 = make_float2(e0_22, e1_22); float2 v2_22; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_204), "l"(*(const unsigned long long*)&_f32x2_mul_b_205)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_236), "l"(*(const unsigned long long*)&_f32x2_mul_b_237)); v_5[4] = v2_22.x; v_5[5] = v2_22.y; float2 d2_63 = sub_f32x2(a2_14, make_float2(gs_2[6], gs_2[7])); float e0_23 = approx_exp2(d2_63.x); float e1_23 = approx_exp2(d2_63.y); - float2 _f32x2_mul_a_206 = make_float2(ks_2[6], ks_2[7]); - float2 _f32x2_mul_b_207 = make_float2(e0_23, e1_23); + float2 _f32x2_mul_a_238 = make_float2(ks_2[6], ks_2[7]); + float2 _f32x2_mul_b_239 = make_float2(e0_23, e1_23); float2 v2_23; - asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_206), "l"(*(const unsigned long long*)&_f32x2_mul_b_207)); + asm("mul.rn.ftz.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&v2_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_238), "l"(*(const unsigned long long*)&_f32x2_mul_b_239)); v_5[6] = v2_23.x; v_5[7] = v2_23.y; asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"((ring_addr + 16384 + (unsigned int)(8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 ^ (8 * th / 32 * 16384 + d_lane * 128 + 8 * th % 32 * 4 >> 7 & 7) << 4))), "f"(v_5[0]), "f"(v_5[1]), "f"(v_5[2]), "f"(v_5[3]) : "memory"); @@ -2844,29 +2963,29 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int kk_40 = w * 4; if (lane_1 > (kk_40 & 15)) { { - uint32_t _addr_208 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_208), "f"(0.0f) : "memory"); + uint32_t _addr_240 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_40 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_240), "f"(0.0f) : "memory"); } } int kk_41 = w * 4 + 1; if (lane_1 > (kk_41 & 15)) { { - uint32_t _addr_209 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_209), "f"(0.0f) : "memory"); + uint32_t _addr_241 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_41 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_241), "f"(0.0f) : "memory"); } } int kk_42 = w * 4 + 2; if (lane_1 > (kk_42 & 15)) { { - uint32_t _addr_210 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_210), "f"(0.0f) : "memory"); + uint32_t _addr_242 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_42 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_242), "f"(0.0f) : "memory"); } } int kk_43 = w * 4 + 3; if (lane_1 > (kk_43 & 15)) { { - uint32_t _addr_211 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_211), "f"(0.0f) : "memory"); + uint32_t _addr_243 = static_cast((dab_addr + (unsigned int)(lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 ^ (lane_1 / 32 * 4096 + kk_43 * 128 + lane_1 % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_243), "f"(0.0f) : "memory"); } } } @@ -2874,29 +2993,29 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int kk_44 = w * 4; if (lane_1 > (kk_44 & 15)) { { - uint32_t _addr_212 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_212), "f"(0.0f) : "memory"); + uint32_t _addr_244 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_44 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_244), "f"(0.0f) : "memory"); } } int kk_45 = w * 4 + 1; if (lane_1 > (kk_45 & 15)) { { - uint32_t _addr_213 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_213), "f"(0.0f) : "memory"); + uint32_t _addr_245 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_45 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_245), "f"(0.0f) : "memory"); } } int kk_46 = w * 4 + 2; if (lane_1 > (kk_46 & 15)) { { - uint32_t _addr_214 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_214), "f"(0.0f) : "memory"); + uint32_t _addr_246 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_46 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_246), "f"(0.0f) : "memory"); } } int kk_47 = w * 4 + 3; if (lane_1 > (kk_47 & 15)) { { - uint32_t _addr_215 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_215), "f"(0.0f) : "memory"); + uint32_t _addr_247 = static_cast((dab_addr + 8192 + (unsigned int)((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 ^ ((16 + lane_1) / 32 * 4096 + kk_47 * 128 + (16 + lane_1) % 32 * 4 >> 7 & 7) << 4))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_247), "f"(0.0f) : "memory"); } } } @@ -2953,9 +3072,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_250[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_0 = __uint_as_float(r_250[0] << 16); float q_t0_0 = ((t_0 >= clen) ? 0.0f : q_t_0); - float dqf_0 = pool_f32[ew + d_lane]; - float dkf_0 = pool_f32[ew + 1024 + d_lane]; - float dgf_0 = pool_f32[ew + 2048 + d_lane]; float bt_t_0 = btv[0]; float e_off_0 = approx_exp2(g_t_0 - gn); float e_dg_0 = approx_exp2(g_t_0 - gc_3); @@ -2963,22 +3079,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_0 = __fmaf_rn(ko[0], e_off_0, kdg[0] * e_dg_0); dbp[0] = dk2_0 * k_t_0; float dk2b_0 = dk2_0 * bt_t_0; + float q_r_0 = q_t0_0; + float qe = q_r_0 * e_dg_0; + float kp = k_t_0 * bt_t_0; + __nv_bfloat16 kb = (__nv_bfloat16)kp; + float kbe = (float)kb * e_dg_0; + float cr = __fmaf_rn(__uint_as_float(__as_u32(qe) & 4294959104u) - qe, qdg[0], (__uint_as_float(__as_u32(kbe) & 4294959104u) - kp * e_dg_0) * kdg[0]); + float dqf_0 = pool_f32[ew + d_lane]; + float dkf_0 = pool_f32[ew + 1024 + d_lane]; + float dgf_0 = pool_f32[ew + 2048 + d_lane]; float dq_o_0 = dq2_0 + dqf_0; float dk_o_0 = dk2b_0 + dkf_0 + dkt_3[0]; - float q_r_0 = q_t0_0; - float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, (dk2b_0 - dkt_3[0]) * k_t_0)) + dgf_0; + float dg_o_0 = __fmaf_rn(q_r_0, dqf_0, __fmaf_rn(q_r_0, dq2_0, __fmaf_rn(dk2b_0 - dkt_3[0], k_t_0, cr))) + dgf_0; dgo[0] = dg_o_0; int qoff_0 = t_0 * qrs; if (t_0 < clen) { { - auto _gst_216 = dq_base + qoff_0; - __builtin_assume(__isGlobal(_gst_216)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_216) + (0)) = (__nv_bfloat16)dq_o_0; + auto _gst_248 = dq_base + qoff_0; + __builtin_assume(__isGlobal(_gst_248)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dq_o_0; } { - auto _gst_217 = dk_base + qoff_0; - __builtin_assume(__isGlobal(_gst_217)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_217) + (0)) = (__nv_bfloat16)dk_o_0; + auto _gst_249 = dk_base + qoff_0; + __builtin_assume(__isGlobal(_gst_249)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dk_o_0; } } int t_1 = 32 + 8 * th + 1; @@ -2992,9 +3116,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_253[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_1 = __uint_as_float(r_253[0] << 16); float q_t0_1 = ((t_1 >= clen) ? 0.0f : q_t_1); - float dqf_1 = pool_f32[ew + 128 + d_lane]; - float dkf_1 = pool_f32[ew + 1152 + d_lane]; - float dgf_1 = pool_f32[ew + 2176 + d_lane]; float bt_t_1 = btv[1]; float e_off_1 = approx_exp2(g_t_1 - gn); float e_dg_1 = approx_exp2(g_t_1 - gc_3); @@ -3002,22 +3123,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_1 = __fmaf_rn(ko[1], e_off_1, kdg[1] * e_dg_1); dbp[1] = dk2_1 * k_t_1; float dk2b_1 = dk2_1 * bt_t_1; + float q_r_1 = q_t0_1; + float qe_0 = q_r_1 * e_dg_1; + float kp_16 = k_t_1 * bt_t_1; + __nv_bfloat16 kb_0 = (__nv_bfloat16)kp_16; + float kbe_0 = (float)kb_0 * e_dg_1; + float cr_0 = __fmaf_rn(__uint_as_float(__as_u32(qe_0) & 4294959104u) - qe_0, qdg[1], (__uint_as_float(__as_u32(kbe_0) & 4294959104u) - kp_16 * e_dg_1) * kdg[1]); + float dqf_1 = pool_f32[ew + 128 + d_lane]; + float dkf_1 = pool_f32[ew + 1152 + d_lane]; + float dgf_1 = pool_f32[ew + 2176 + d_lane]; float dq_o_1 = dq2_1 + dqf_1; float dk_o_1 = dk2b_1 + dkf_1 + dkt_3[1]; - float q_r_1 = q_t0_1; - float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, (dk2b_1 - dkt_3[1]) * k_t_1)) + dgf_1; + float dg_o_1 = __fmaf_rn(q_r_1, dqf_1, __fmaf_rn(q_r_1, dq2_1, __fmaf_rn(dk2b_1 - dkt_3[1], k_t_1, cr_0))) + dgf_1; dgo[1] = dg_o_1; int qoff_1 = t_1 * qrs; if (t_1 < clen) { { - auto _gst_218 = dq_base + qoff_1; - __builtin_assume(__isGlobal(_gst_218)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_218) + (0)) = (__nv_bfloat16)dq_o_1; + auto _gst_250 = dq_base + qoff_1; + __builtin_assume(__isGlobal(_gst_250)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dq_o_1; } { - auto _gst_219 = dk_base + qoff_1; - __builtin_assume(__isGlobal(_gst_219)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_219) + (0)) = (__nv_bfloat16)dk_o_1; + auto _gst_251 = dk_base + qoff_1; + __builtin_assume(__isGlobal(_gst_251)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dk_o_1; } } int t_2 = 32 + 8 * th + 2; @@ -3031,9 +3160,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_256[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_2 = __uint_as_float(r_256[0] << 16); float q_t0_2 = ((t_2 >= clen) ? 0.0f : q_t_2); - float dqf_2 = pool_f32[ew + 256 + d_lane]; - float dkf_2 = pool_f32[ew + 1280 + d_lane]; - float dgf_2 = pool_f32[ew + 2304 + d_lane]; float bt_t_2 = btv[2]; float e_off_2 = approx_exp2(g_t_2 - gn); float e_dg_2 = approx_exp2(g_t_2 - gc_3); @@ -3041,22 +3167,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_2 = __fmaf_rn(ko[2], e_off_2, kdg[2] * e_dg_2); dbp[2] = dk2_2 * k_t_2; float dk2b_2 = dk2_2 * bt_t_2; + float q_r_2 = q_t0_2; + float qe_1 = q_r_2 * e_dg_2; + float kp_17 = k_t_2 * bt_t_2; + __nv_bfloat16 kb_1 = (__nv_bfloat16)kp_17; + float kbe_1 = (float)kb_1 * e_dg_2; + float cr_1 = __fmaf_rn(__uint_as_float(__as_u32(qe_1) & 4294959104u) - qe_1, qdg[2], (__uint_as_float(__as_u32(kbe_1) & 4294959104u) - kp_17 * e_dg_2) * kdg[2]); + float dqf_2 = pool_f32[ew + 256 + d_lane]; + float dkf_2 = pool_f32[ew + 1280 + d_lane]; + float dgf_2 = pool_f32[ew + 2304 + d_lane]; float dq_o_2 = dq2_2 + dqf_2; float dk_o_2 = dk2b_2 + dkf_2 + dkt_3[2]; - float q_r_2 = q_t0_2; - float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, (dk2b_2 - dkt_3[2]) * k_t_2)) + dgf_2; + float dg_o_2 = __fmaf_rn(q_r_2, dqf_2, __fmaf_rn(q_r_2, dq2_2, __fmaf_rn(dk2b_2 - dkt_3[2], k_t_2, cr_1))) + dgf_2; dgo[2] = dg_o_2; int qoff_2 = t_2 * qrs; if (t_2 < clen) { { - auto _gst_220 = dq_base + qoff_2; - __builtin_assume(__isGlobal(_gst_220)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_220) + (0)) = (__nv_bfloat16)dq_o_2; + auto _gst_252 = dq_base + qoff_2; + __builtin_assume(__isGlobal(_gst_252)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dq_o_2; } { - auto _gst_221 = dk_base + qoff_2; - __builtin_assume(__isGlobal(_gst_221)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_221) + (0)) = (__nv_bfloat16)dk_o_2; + auto _gst_253 = dk_base + qoff_2; + __builtin_assume(__isGlobal(_gst_253)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dk_o_2; } } int t_3 = 32 + 8 * th + 3; @@ -3070,9 +3204,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_259[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_3 = __uint_as_float(r_259[0] << 16); float q_t0_3 = ((t_3 >= clen) ? 0.0f : q_t_3); - float dqf_3 = pool_f32[ew + 384 + d_lane]; - float dkf_3 = pool_f32[ew + 1408 + d_lane]; - float dgf_3 = pool_f32[ew + 2432 + d_lane]; float bt_t_3 = btv[3]; float e_off_3 = approx_exp2(g_t_3 - gn); float e_dg_3 = approx_exp2(g_t_3 - gc_3); @@ -3080,22 +3211,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_3 = __fmaf_rn(ko[3], e_off_3, kdg[3] * e_dg_3); dbp[3] = dk2_3 * k_t_3; float dk2b_3 = dk2_3 * bt_t_3; + float q_r_3 = q_t0_3; + float qe_2 = q_r_3 * e_dg_3; + float kp_18 = k_t_3 * bt_t_3; + __nv_bfloat16 kb_2 = (__nv_bfloat16)kp_18; + float kbe_2 = (float)kb_2 * e_dg_3; + float cr_2 = __fmaf_rn(__uint_as_float(__as_u32(qe_2) & 4294959104u) - qe_2, qdg[3], (__uint_as_float(__as_u32(kbe_2) & 4294959104u) - kp_18 * e_dg_3) * kdg[3]); + float dqf_3 = pool_f32[ew + 384 + d_lane]; + float dkf_3 = pool_f32[ew + 1408 + d_lane]; + float dgf_3 = pool_f32[ew + 2432 + d_lane]; float dq_o_3 = dq2_3 + dqf_3; float dk_o_3 = dk2b_3 + dkf_3 + dkt_3[3]; - float q_r_3 = q_t0_3; - float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, (dk2b_3 - dkt_3[3]) * k_t_3)) + dgf_3; + float dg_o_3 = __fmaf_rn(q_r_3, dqf_3, __fmaf_rn(q_r_3, dq2_3, __fmaf_rn(dk2b_3 - dkt_3[3], k_t_3, cr_2))) + dgf_3; dgo[3] = dg_o_3; int qoff_3 = t_3 * qrs; if (t_3 < clen) { { - auto _gst_222 = dq_base + qoff_3; - __builtin_assume(__isGlobal(_gst_222)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_222) + (0)) = (__nv_bfloat16)dq_o_3; + auto _gst_254 = dq_base + qoff_3; + __builtin_assume(__isGlobal(_gst_254)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dq_o_3; } { - auto _gst_223 = dk_base + qoff_3; - __builtin_assume(__isGlobal(_gst_223)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_223) + (0)) = (__nv_bfloat16)dk_o_3; + auto _gst_255 = dk_base + qoff_3; + __builtin_assume(__isGlobal(_gst_255)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dk_o_3; } } int t_4 = 32 + 8 * th + 4; @@ -3109,9 +3248,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_262[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_4 = __uint_as_float(r_262[0] << 16); float q_t0_4 = ((t_4 >= clen) ? 0.0f : q_t_4); - float dqf_4 = pool_f32[ew + 512 + d_lane]; - float dkf_4 = pool_f32[ew + 1536 + d_lane]; - float dgf_4 = pool_f32[ew + 2560 + d_lane]; float bt_t_4 = btv[4]; float e_off_4 = approx_exp2(g_t_4 - gn); float e_dg_4 = approx_exp2(g_t_4 - gc_3); @@ -3119,22 +3255,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_4 = __fmaf_rn(ko[4], e_off_4, kdg[4] * e_dg_4); dbp[4] = dk2_4 * k_t_4; float dk2b_4 = dk2_4 * bt_t_4; + float q_r_4 = q_t0_4; + float qe_3 = q_r_4 * e_dg_4; + float kp_19 = k_t_4 * bt_t_4; + __nv_bfloat16 kb_3 = (__nv_bfloat16)kp_19; + float kbe_3 = (float)kb_3 * e_dg_4; + float cr_3 = __fmaf_rn(__uint_as_float(__as_u32(qe_3) & 4294959104u) - qe_3, qdg[4], (__uint_as_float(__as_u32(kbe_3) & 4294959104u) - kp_19 * e_dg_4) * kdg[4]); + float dqf_4 = pool_f32[ew + 512 + d_lane]; + float dkf_4 = pool_f32[ew + 1536 + d_lane]; + float dgf_4 = pool_f32[ew + 2560 + d_lane]; float dq_o_4 = dq2_4 + dqf_4; float dk_o_4 = dk2b_4 + dkf_4 + dkt_3[4]; - float q_r_4 = q_t0_4; - float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, (dk2b_4 - dkt_3[4]) * k_t_4)) + dgf_4; + float dg_o_4 = __fmaf_rn(q_r_4, dqf_4, __fmaf_rn(q_r_4, dq2_4, __fmaf_rn(dk2b_4 - dkt_3[4], k_t_4, cr_3))) + dgf_4; dgo[4] = dg_o_4; int qoff_4 = t_4 * qrs; if (t_4 < clen) { { - auto _gst_224 = dq_base + qoff_4; - __builtin_assume(__isGlobal(_gst_224)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_224) + (0)) = (__nv_bfloat16)dq_o_4; + auto _gst_256 = dq_base + qoff_4; + __builtin_assume(__isGlobal(_gst_256)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dq_o_4; } { - auto _gst_225 = dk_base + qoff_4; - __builtin_assume(__isGlobal(_gst_225)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_225) + (0)) = (__nv_bfloat16)dk_o_4; + auto _gst_257 = dk_base + qoff_4; + __builtin_assume(__isGlobal(_gst_257)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_257) + (0)) = (__nv_bfloat16)dk_o_4; } } int t_5 = 32 + 8 * th + 5; @@ -3148,9 +3292,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_265[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_5 = __uint_as_float(r_265[0] << 16); float q_t0_5 = ((t_5 >= clen) ? 0.0f : q_t_5); - float dqf_5 = pool_f32[ew + 640 + d_lane]; - float dkf_5 = pool_f32[ew + 1664 + d_lane]; - float dgf_5 = pool_f32[ew + 2688 + d_lane]; float bt_t_5 = btv[5]; float e_off_5 = approx_exp2(g_t_5 - gn); float e_dg_5 = approx_exp2(g_t_5 - gc_3); @@ -3158,22 +3299,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_5 = __fmaf_rn(ko[5], e_off_5, kdg[5] * e_dg_5); dbp[5] = dk2_5 * k_t_5; float dk2b_5 = dk2_5 * bt_t_5; + float q_r_5 = q_t0_5; + float qe_4 = q_r_5 * e_dg_5; + float kp_20 = k_t_5 * bt_t_5; + __nv_bfloat16 kb_4 = (__nv_bfloat16)kp_20; + float kbe_4 = (float)kb_4 * e_dg_5; + float cr_4 = __fmaf_rn(__uint_as_float(__as_u32(qe_4) & 4294959104u) - qe_4, qdg[5], (__uint_as_float(__as_u32(kbe_4) & 4294959104u) - kp_20 * e_dg_5) * kdg[5]); + float dqf_5 = pool_f32[ew + 640 + d_lane]; + float dkf_5 = pool_f32[ew + 1664 + d_lane]; + float dgf_5 = pool_f32[ew + 2688 + d_lane]; float dq_o_5 = dq2_5 + dqf_5; float dk_o_5 = dk2b_5 + dkf_5 + dkt_3[5]; - float q_r_5 = q_t0_5; - float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, (dk2b_5 - dkt_3[5]) * k_t_5)) + dgf_5; + float dg_o_5 = __fmaf_rn(q_r_5, dqf_5, __fmaf_rn(q_r_5, dq2_5, __fmaf_rn(dk2b_5 - dkt_3[5], k_t_5, cr_4))) + dgf_5; dgo[5] = dg_o_5; int qoff_5 = t_5 * qrs; if (t_5 < clen) { { - auto _gst_226 = dq_base + qoff_5; - __builtin_assume(__isGlobal(_gst_226)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_226) + (0)) = (__nv_bfloat16)dq_o_5; + auto _gst_258 = dq_base + qoff_5; + __builtin_assume(__isGlobal(_gst_258)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_258) + (0)) = (__nv_bfloat16)dq_o_5; } { - auto _gst_227 = dk_base + qoff_5; - __builtin_assume(__isGlobal(_gst_227)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_227) + (0)) = (__nv_bfloat16)dk_o_5; + auto _gst_259 = dk_base + qoff_5; + __builtin_assume(__isGlobal(_gst_259)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_259) + (0)) = (__nv_bfloat16)dk_o_5; } } int t_6 = 32 + 8 * th + 6; @@ -3187,9 +3336,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_268[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_6 = __uint_as_float(r_268[0] << 16); float q_t0_6 = ((t_6 >= clen) ? 0.0f : q_t_6); - float dqf_6 = pool_f32[ew + 768 + d_lane]; - float dkf_6 = pool_f32[ew + 1792 + d_lane]; - float dgf_6 = pool_f32[ew + 2816 + d_lane]; float bt_t_6 = btv[6]; float e_off_6 = approx_exp2(g_t_6 - gn); float e_dg_6 = approx_exp2(g_t_6 - gc_3); @@ -3197,22 +3343,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_6 = __fmaf_rn(ko[6], e_off_6, kdg[6] * e_dg_6); dbp[6] = dk2_6 * k_t_6; float dk2b_6 = dk2_6 * bt_t_6; + float q_r_6 = q_t0_6; + float qe_5 = q_r_6 * e_dg_6; + float kp_21 = k_t_6 * bt_t_6; + __nv_bfloat16 kb_5 = (__nv_bfloat16)kp_21; + float kbe_5 = (float)kb_5 * e_dg_6; + float cr_5 = __fmaf_rn(__uint_as_float(__as_u32(qe_5) & 4294959104u) - qe_5, qdg[6], (__uint_as_float(__as_u32(kbe_5) & 4294959104u) - kp_21 * e_dg_6) * kdg[6]); + float dqf_6 = pool_f32[ew + 768 + d_lane]; + float dkf_6 = pool_f32[ew + 1792 + d_lane]; + float dgf_6 = pool_f32[ew + 2816 + d_lane]; float dq_o_6 = dq2_6 + dqf_6; float dk_o_6 = dk2b_6 + dkf_6 + dkt_3[6]; - float q_r_6 = q_t0_6; - float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, (dk2b_6 - dkt_3[6]) * k_t_6)) + dgf_6; + float dg_o_6 = __fmaf_rn(q_r_6, dqf_6, __fmaf_rn(q_r_6, dq2_6, __fmaf_rn(dk2b_6 - dkt_3[6], k_t_6, cr_5))) + dgf_6; dgo[6] = dg_o_6; int qoff_6 = t_6 * qrs; if (t_6 < clen) { { - auto _gst_228 = dq_base + qoff_6; - __builtin_assume(__isGlobal(_gst_228)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_228) + (0)) = (__nv_bfloat16)dq_o_6; + auto _gst_260 = dq_base + qoff_6; + __builtin_assume(__isGlobal(_gst_260)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_260) + (0)) = (__nv_bfloat16)dq_o_6; } { - auto _gst_229 = dk_base + qoff_6; - __builtin_assume(__isGlobal(_gst_229)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_229) + (0)) = (__nv_bfloat16)dk_o_6; + auto _gst_261 = dk_base + qoff_6; + __builtin_assume(__isGlobal(_gst_261)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_261) + (0)) = (__nv_bfloat16)dk_o_6; } } int t_7 = 32 + 8 * th + 7; @@ -3226,9 +3380,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_271[0])) : "r"((qbase_3 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_7 = __uint_as_float(r_271[0] << 16); float q_t0_7 = ((t_7 >= clen) ? 0.0f : q_t_7); - float dqf_7 = pool_f32[ew + 896 + d_lane]; - float dkf_7 = pool_f32[ew + 1920 + d_lane]; - float dgf_7 = pool_f32[ew + 2944 + d_lane]; float bt_t_7 = btv[7]; float e_off_7 = approx_exp2(g_t_7 - gn); float e_dg_7 = approx_exp2(g_t_7 - gc_3); @@ -3236,22 +3387,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_7 = __fmaf_rn(ko[7], e_off_7, kdg[7] * e_dg_7); dbp[7] = dk2_7 * k_t_7; float dk2b_7 = dk2_7 * bt_t_7; + float q_r_7 = q_t0_7; + float qe_6 = q_r_7 * e_dg_7; + float kp_22 = k_t_7 * bt_t_7; + __nv_bfloat16 kb_6 = (__nv_bfloat16)kp_22; + float kbe_6 = (float)kb_6 * e_dg_7; + float cr_6 = __fmaf_rn(__uint_as_float(__as_u32(qe_6) & 4294959104u) - qe_6, qdg[7], (__uint_as_float(__as_u32(kbe_6) & 4294959104u) - kp_22 * e_dg_7) * kdg[7]); + float dqf_7 = pool_f32[ew + 896 + d_lane]; + float dkf_7 = pool_f32[ew + 1920 + d_lane]; + float dgf_7 = pool_f32[ew + 2944 + d_lane]; float dq_o_7 = dq2_7 + dqf_7; float dk_o_7 = dk2b_7 + dkf_7 + dkt_3[7]; - float q_r_7 = q_t0_7; - float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, (dk2b_7 - dkt_3[7]) * k_t_7)) + dgf_7; + float dg_o_7 = __fmaf_rn(q_r_7, dqf_7, __fmaf_rn(q_r_7, dq2_7, __fmaf_rn(dk2b_7 - dkt_3[7], k_t_7, cr_6))) + dgf_7; dgo[7] = dg_o_7; int qoff_7 = t_7 * qrs; if (t_7 < clen) { { - auto _gst_230 = dq_base + qoff_7; - __builtin_assume(__isGlobal(_gst_230)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_230) + (0)) = (__nv_bfloat16)dq_o_7; + auto _gst_262 = dq_base + qoff_7; + __builtin_assume(__isGlobal(_gst_262)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_262) + (0)) = (__nv_bfloat16)dq_o_7; } { - auto _gst_231 = dk_base + qoff_7; - __builtin_assume(__isGlobal(_gst_231)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_231) + (0)) = (__nv_bfloat16)dk_o_7; + auto _gst_263 = dk_base + qoff_7; + __builtin_assume(__isGlobal(_gst_263)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_263) + (0)) = (__nv_bfloat16)dk_o_7; } } float s_run[1]; @@ -3280,31 +3439,31 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int b2 = lane_1 >> 2 & 1; float v4[4]; float send_0 = ((b4 == 0) ? dbp[4] : dbp[0]); - float _shfl_xor_sync_232 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); - v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_232; + float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, send_0, 16); + v4[0] = ((b4 == 0) ? dbp[0] : dbp[4]) + _shfl_xor_sync_264; float send_1 = ((b4 == 0) ? dbp[5] : dbp[1]); - float _shfl_xor_sync_233 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); - v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_233; + float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, send_1, 16); + v4[1] = ((b4 == 0) ? dbp[1] : dbp[5]) + _shfl_xor_sync_265; float send_2 = ((b4 == 0) ? dbp[6] : dbp[2]); - float _shfl_xor_sync_234 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); - v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_234; + float _shfl_xor_sync_266 = __shfl_xor_sync(0xFFFFFFFF, send_2, 16); + v4[2] = ((b4 == 0) ? dbp[2] : dbp[6]) + _shfl_xor_sync_266; float send_3 = ((b4 == 0) ? dbp[7] : dbp[3]); - float _shfl_xor_sync_235 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); - v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_235; + float _shfl_xor_sync_267 = __shfl_xor_sync(0xFFFFFFFF, send_3, 16); + v4[3] = ((b4 == 0) ? dbp[3] : dbp[7]) + _shfl_xor_sync_267; float v2[2]; float send_4 = ((b3 == 0) ? v4[2] : v4[0]); - float _shfl_xor_sync_236 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); - v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_236; + float _shfl_xor_sync_268 = __shfl_xor_sync(0xFFFFFFFF, send_4, 8); + v2[0] = ((b3 == 0) ? v4[0] : v4[2]) + _shfl_xor_sync_268; float send_5 = ((b3 == 0) ? v4[3] : v4[1]); - float _shfl_xor_sync_237 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); - v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_237; + float _shfl_xor_sync_269 = __shfl_xor_sync(0xFFFFFFFF, send_5, 8); + v2[1] = ((b3 == 0) ? v4[1] : v4[3]) + _shfl_xor_sync_269; float send = ((b2 == 0) ? v2[1] : v2[0]); - float _shfl_xor_sync_238 = __shfl_xor_sync(0xFFFFFFFF, send, 4); - float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_238; - float _shfl_xor_sync_239 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); - v1 = v1 + _shfl_xor_sync_239; - float _shfl_xor_sync_240 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); - v1 = v1 + _shfl_xor_sync_240; + float _shfl_xor_sync_270 = __shfl_xor_sync(0xFFFFFFFF, send, 4); + float v1 = ((b2 == 0) ? v2[0] : v2[1]) + _shfl_xor_sync_270; + float _shfl_xor_sync_271 = __shfl_xor_sync(0xFFFFFFFF, v1, 2); + v1 = v1 + _shfl_xor_sync_271; + float _shfl_xor_sync_272 = __shfl_xor_sync(0xFFFFFFFF, v1, 1); + v1 = v1 + _shfl_xor_sync_272; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(p1a_t2_drained_addr); mbarrier_arrive(epi_empty_addr + (th) * 8); @@ -3351,9 +3510,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_275[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_8 = __uint_as_float(r_275[0] << 16); float q_t0_8 = ((t_8 >= clen) ? 0.0f : q_t_8); - float dqf_8 = pool_f32[ew_0 + d_lane]; - float dkf_8 = pool_f32[ew_0 + 1024 + d_lane]; - float dgf_8 = pool_f32[ew_0 + 2048 + d_lane]; float bt_t_8 = btv_0[0]; float e_off_8 = approx_exp2(g_t_8 - gn_0); float e_dg_8 = approx_exp2(g_t_8 - gc_4); @@ -3361,22 +3517,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_8 = __fmaf_rn(ko_0[0], e_off_8, kdg_0[0] * e_dg_8); dbp_0[0] = dk2_8 * k_t_8; float dk2b_8 = dk2_8 * bt_t_8; + float q_r_8 = q_t0_8; + float qe_7 = q_r_8 * e_dg_8; + float kp_23 = k_t_8 * bt_t_8; + __nv_bfloat16 kb_7 = (__nv_bfloat16)kp_23; + float kbe_7 = (float)kb_7 * e_dg_8; + float cr_7 = __fmaf_rn(__uint_as_float(__as_u32(qe_7) & 4294959104u) - qe_7, qdg_0[0], (__uint_as_float(__as_u32(kbe_7) & 4294959104u) - kp_23 * e_dg_8) * kdg_0[0]); + float dqf_8 = pool_f32[ew_0 + d_lane]; + float dkf_8 = pool_f32[ew_0 + 1024 + d_lane]; + float dgf_8 = pool_f32[ew_0 + 2048 + d_lane]; float dq_o_8 = dq2_8 + dqf_8; float dk_o_8 = dk2b_8 + dkf_8 + dkt_4[0]; - float q_r_8 = q_t0_8; - float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, (dk2b_8 - dkt_4[0]) * k_t_8)) + dgf_8; + float dg_o_8 = __fmaf_rn(q_r_8, dqf_8, __fmaf_rn(q_r_8, dq2_8, __fmaf_rn(dk2b_8 - dkt_4[0], k_t_8, cr_7))) + dgf_8; dgo_0[0] = dg_o_8; int qoff_8 = t_8 * qrs; if (t_8 < clen) { { - auto _gst_241 = dq_base + qoff_8; - __builtin_assume(__isGlobal(_gst_241)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_241) + (0)) = (__nv_bfloat16)dq_o_8; + auto _gst_273 = dq_base + qoff_8; + __builtin_assume(__isGlobal(_gst_273)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_273) + (0)) = (__nv_bfloat16)dq_o_8; } { - auto _gst_242 = dk_base + qoff_8; - __builtin_assume(__isGlobal(_gst_242)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_242) + (0)) = (__nv_bfloat16)dk_o_8; + auto _gst_274 = dk_base + qoff_8; + __builtin_assume(__isGlobal(_gst_274)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_274) + (0)) = (__nv_bfloat16)dk_o_8; } } int t_9 = 48 + 8 * th + 1; @@ -3390,9 +3554,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_278[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_9 = __uint_as_float(r_278[0] << 16); float q_t0_9 = ((t_9 >= clen) ? 0.0f : q_t_9); - float dqf_9 = pool_f32[ew_0 + 128 + d_lane]; - float dkf_9 = pool_f32[ew_0 + 1152 + d_lane]; - float dgf_9 = pool_f32[ew_0 + 2176 + d_lane]; float bt_t_9 = btv_0[1]; float e_off_9 = approx_exp2(g_t_9 - gn_0); float e_dg_9 = approx_exp2(g_t_9 - gc_4); @@ -3400,22 +3561,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_9 = __fmaf_rn(ko_0[1], e_off_9, kdg_0[1] * e_dg_9); dbp_0[1] = dk2_9 * k_t_9; float dk2b_9 = dk2_9 * bt_t_9; + float q_r_9 = q_t0_9; + float qe_8 = q_r_9 * e_dg_9; + float kp_24 = k_t_9 * bt_t_9; + __nv_bfloat16 kb_8 = (__nv_bfloat16)kp_24; + float kbe_8 = (float)kb_8 * e_dg_9; + float cr_8 = __fmaf_rn(__uint_as_float(__as_u32(qe_8) & 4294959104u) - qe_8, qdg_0[1], (__uint_as_float(__as_u32(kbe_8) & 4294959104u) - kp_24 * e_dg_9) * kdg_0[1]); + float dqf_9 = pool_f32[ew_0 + 128 + d_lane]; + float dkf_9 = pool_f32[ew_0 + 1152 + d_lane]; + float dgf_9 = pool_f32[ew_0 + 2176 + d_lane]; float dq_o_9 = dq2_9 + dqf_9; float dk_o_9 = dk2b_9 + dkf_9 + dkt_4[1]; - float q_r_9 = q_t0_9; - float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, (dk2b_9 - dkt_4[1]) * k_t_9)) + dgf_9; + float dg_o_9 = __fmaf_rn(q_r_9, dqf_9, __fmaf_rn(q_r_9, dq2_9, __fmaf_rn(dk2b_9 - dkt_4[1], k_t_9, cr_8))) + dgf_9; dgo_0[1] = dg_o_9; int qoff_9 = t_9 * qrs; if (t_9 < clen) { { - auto _gst_243 = dq_base + qoff_9; - __builtin_assume(__isGlobal(_gst_243)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_243) + (0)) = (__nv_bfloat16)dq_o_9; + auto _gst_275 = dq_base + qoff_9; + __builtin_assume(__isGlobal(_gst_275)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_275) + (0)) = (__nv_bfloat16)dq_o_9; } { - auto _gst_244 = dk_base + qoff_9; - __builtin_assume(__isGlobal(_gst_244)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_244) + (0)) = (__nv_bfloat16)dk_o_9; + auto _gst_276 = dk_base + qoff_9; + __builtin_assume(__isGlobal(_gst_276)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_276) + (0)) = (__nv_bfloat16)dk_o_9; } } int t_10 = 48 + 8 * th + 2; @@ -3429,9 +3598,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_281[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_10 = __uint_as_float(r_281[0] << 16); float q_t0_10 = ((t_10 >= clen) ? 0.0f : q_t_10); - float dqf_10 = pool_f32[ew_0 + 256 + d_lane]; - float dkf_10 = pool_f32[ew_0 + 1280 + d_lane]; - float dgf_10 = pool_f32[ew_0 + 2304 + d_lane]; float bt_t_10 = btv_0[2]; float e_off_10 = approx_exp2(g_t_10 - gn_0); float e_dg_10 = approx_exp2(g_t_10 - gc_4); @@ -3439,22 +3605,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_10 = __fmaf_rn(ko_0[2], e_off_10, kdg_0[2] * e_dg_10); dbp_0[2] = dk2_10 * k_t_10; float dk2b_10 = dk2_10 * bt_t_10; + float q_r_10 = q_t0_10; + float qe_9 = q_r_10 * e_dg_10; + float kp_25 = k_t_10 * bt_t_10; + __nv_bfloat16 kb_9 = (__nv_bfloat16)kp_25; + float kbe_9 = (float)kb_9 * e_dg_10; + float cr_9 = __fmaf_rn(__uint_as_float(__as_u32(qe_9) & 4294959104u) - qe_9, qdg_0[2], (__uint_as_float(__as_u32(kbe_9) & 4294959104u) - kp_25 * e_dg_10) * kdg_0[2]); + float dqf_10 = pool_f32[ew_0 + 256 + d_lane]; + float dkf_10 = pool_f32[ew_0 + 1280 + d_lane]; + float dgf_10 = pool_f32[ew_0 + 2304 + d_lane]; float dq_o_10 = dq2_10 + dqf_10; float dk_o_10 = dk2b_10 + dkf_10 + dkt_4[2]; - float q_r_10 = q_t0_10; - float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, (dk2b_10 - dkt_4[2]) * k_t_10)) + dgf_10; + float dg_o_10 = __fmaf_rn(q_r_10, dqf_10, __fmaf_rn(q_r_10, dq2_10, __fmaf_rn(dk2b_10 - dkt_4[2], k_t_10, cr_9))) + dgf_10; dgo_0[2] = dg_o_10; int qoff_10 = t_10 * qrs; if (t_10 < clen) { { - auto _gst_245 = dq_base + qoff_10; - __builtin_assume(__isGlobal(_gst_245)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_245) + (0)) = (__nv_bfloat16)dq_o_10; + auto _gst_277 = dq_base + qoff_10; + __builtin_assume(__isGlobal(_gst_277)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_277) + (0)) = (__nv_bfloat16)dq_o_10; } { - auto _gst_246 = dk_base + qoff_10; - __builtin_assume(__isGlobal(_gst_246)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_246) + (0)) = (__nv_bfloat16)dk_o_10; + auto _gst_278 = dk_base + qoff_10; + __builtin_assume(__isGlobal(_gst_278)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_278) + (0)) = (__nv_bfloat16)dk_o_10; } } int t_11 = 48 + 8 * th + 3; @@ -3468,9 +3642,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_284[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_11 = __uint_as_float(r_284[0] << 16); float q_t0_11 = ((t_11 >= clen) ? 0.0f : q_t_11); - float dqf_11 = pool_f32[ew_0 + 384 + d_lane]; - float dkf_11 = pool_f32[ew_0 + 1408 + d_lane]; - float dgf_11 = pool_f32[ew_0 + 2432 + d_lane]; float bt_t_11 = btv_0[3]; float e_off_11 = approx_exp2(g_t_11 - gn_0); float e_dg_11 = approx_exp2(g_t_11 - gc_4); @@ -3478,22 +3649,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_11 = __fmaf_rn(ko_0[3], e_off_11, kdg_0[3] * e_dg_11); dbp_0[3] = dk2_11 * k_t_11; float dk2b_11 = dk2_11 * bt_t_11; + float q_r_11 = q_t0_11; + float qe_10 = q_r_11 * e_dg_11; + float kp_26 = k_t_11 * bt_t_11; + __nv_bfloat16 kb_10 = (__nv_bfloat16)kp_26; + float kbe_10 = (float)kb_10 * e_dg_11; + float cr_10 = __fmaf_rn(__uint_as_float(__as_u32(qe_10) & 4294959104u) - qe_10, qdg_0[3], (__uint_as_float(__as_u32(kbe_10) & 4294959104u) - kp_26 * e_dg_11) * kdg_0[3]); + float dqf_11 = pool_f32[ew_0 + 384 + d_lane]; + float dkf_11 = pool_f32[ew_0 + 1408 + d_lane]; + float dgf_11 = pool_f32[ew_0 + 2432 + d_lane]; float dq_o_11 = dq2_11 + dqf_11; float dk_o_11 = dk2b_11 + dkf_11 + dkt_4[3]; - float q_r_11 = q_t0_11; - float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, (dk2b_11 - dkt_4[3]) * k_t_11)) + dgf_11; + float dg_o_11 = __fmaf_rn(q_r_11, dqf_11, __fmaf_rn(q_r_11, dq2_11, __fmaf_rn(dk2b_11 - dkt_4[3], k_t_11, cr_10))) + dgf_11; dgo_0[3] = dg_o_11; int qoff_11 = t_11 * qrs; if (t_11 < clen) { { - auto _gst_247 = dq_base + qoff_11; - __builtin_assume(__isGlobal(_gst_247)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_247) + (0)) = (__nv_bfloat16)dq_o_11; + auto _gst_279 = dq_base + qoff_11; + __builtin_assume(__isGlobal(_gst_279)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_279) + (0)) = (__nv_bfloat16)dq_o_11; } { - auto _gst_248 = dk_base + qoff_11; - __builtin_assume(__isGlobal(_gst_248)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_248) + (0)) = (__nv_bfloat16)dk_o_11; + auto _gst_280 = dk_base + qoff_11; + __builtin_assume(__isGlobal(_gst_280)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_280) + (0)) = (__nv_bfloat16)dk_o_11; } } int t_12 = 48 + 8 * th + 4; @@ -3507,9 +3686,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_287[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_12 = __uint_as_float(r_287[0] << 16); float q_t0_12 = ((t_12 >= clen) ? 0.0f : q_t_12); - float dqf_12 = pool_f32[ew_0 + 512 + d_lane]; - float dkf_12 = pool_f32[ew_0 + 1536 + d_lane]; - float dgf_12 = pool_f32[ew_0 + 2560 + d_lane]; float bt_t_12 = btv_0[4]; float e_off_12 = approx_exp2(g_t_12 - gn_0); float e_dg_12 = approx_exp2(g_t_12 - gc_4); @@ -3517,22 +3693,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_12 = __fmaf_rn(ko_0[4], e_off_12, kdg_0[4] * e_dg_12); dbp_0[4] = dk2_12 * k_t_12; float dk2b_12 = dk2_12 * bt_t_12; + float q_r_12 = q_t0_12; + float qe_11 = q_r_12 * e_dg_12; + float kp_27 = k_t_12 * bt_t_12; + __nv_bfloat16 kb_11 = (__nv_bfloat16)kp_27; + float kbe_11 = (float)kb_11 * e_dg_12; + float cr_11 = __fmaf_rn(__uint_as_float(__as_u32(qe_11) & 4294959104u) - qe_11, qdg_0[4], (__uint_as_float(__as_u32(kbe_11) & 4294959104u) - kp_27 * e_dg_12) * kdg_0[4]); + float dqf_12 = pool_f32[ew_0 + 512 + d_lane]; + float dkf_12 = pool_f32[ew_0 + 1536 + d_lane]; + float dgf_12 = pool_f32[ew_0 + 2560 + d_lane]; float dq_o_12 = dq2_12 + dqf_12; float dk_o_12 = dk2b_12 + dkf_12 + dkt_4[4]; - float q_r_12 = q_t0_12; - float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, (dk2b_12 - dkt_4[4]) * k_t_12)) + dgf_12; + float dg_o_12 = __fmaf_rn(q_r_12, dqf_12, __fmaf_rn(q_r_12, dq2_12, __fmaf_rn(dk2b_12 - dkt_4[4], k_t_12, cr_11))) + dgf_12; dgo_0[4] = dg_o_12; int qoff_12 = t_12 * qrs; if (t_12 < clen) { { - auto _gst_249 = dq_base + qoff_12; - __builtin_assume(__isGlobal(_gst_249)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_249) + (0)) = (__nv_bfloat16)dq_o_12; + auto _gst_281 = dq_base + qoff_12; + __builtin_assume(__isGlobal(_gst_281)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_281) + (0)) = (__nv_bfloat16)dq_o_12; } { - auto _gst_250 = dk_base + qoff_12; - __builtin_assume(__isGlobal(_gst_250)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_250) + (0)) = (__nv_bfloat16)dk_o_12; + auto _gst_282 = dk_base + qoff_12; + __builtin_assume(__isGlobal(_gst_282)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_282) + (0)) = (__nv_bfloat16)dk_o_12; } } int t_13 = 48 + 8 * th + 5; @@ -3546,9 +3730,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_290[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_13 = __uint_as_float(r_290[0] << 16); float q_t0_13 = ((t_13 >= clen) ? 0.0f : q_t_13); - float dqf_13 = pool_f32[ew_0 + 640 + d_lane]; - float dkf_13 = pool_f32[ew_0 + 1664 + d_lane]; - float dgf_13 = pool_f32[ew_0 + 2688 + d_lane]; float bt_t_13 = btv_0[5]; float e_off_13 = approx_exp2(g_t_13 - gn_0); float e_dg_13 = approx_exp2(g_t_13 - gc_4); @@ -3556,22 +3737,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_13 = __fmaf_rn(ko_0[5], e_off_13, kdg_0[5] * e_dg_13); dbp_0[5] = dk2_13 * k_t_13; float dk2b_13 = dk2_13 * bt_t_13; + float q_r_13 = q_t0_13; + float qe_12 = q_r_13 * e_dg_13; + float kp_28 = k_t_13 * bt_t_13; + __nv_bfloat16 kb_12 = (__nv_bfloat16)kp_28; + float kbe_12 = (float)kb_12 * e_dg_13; + float cr_12 = __fmaf_rn(__uint_as_float(__as_u32(qe_12) & 4294959104u) - qe_12, qdg_0[5], (__uint_as_float(__as_u32(kbe_12) & 4294959104u) - kp_28 * e_dg_13) * kdg_0[5]); + float dqf_13 = pool_f32[ew_0 + 640 + d_lane]; + float dkf_13 = pool_f32[ew_0 + 1664 + d_lane]; + float dgf_13 = pool_f32[ew_0 + 2688 + d_lane]; float dq_o_13 = dq2_13 + dqf_13; float dk_o_13 = dk2b_13 + dkf_13 + dkt_4[5]; - float q_r_13 = q_t0_13; - float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, (dk2b_13 - dkt_4[5]) * k_t_13)) + dgf_13; + float dg_o_13 = __fmaf_rn(q_r_13, dqf_13, __fmaf_rn(q_r_13, dq2_13, __fmaf_rn(dk2b_13 - dkt_4[5], k_t_13, cr_12))) + dgf_13; dgo_0[5] = dg_o_13; int qoff_13 = t_13 * qrs; if (t_13 < clen) { { - auto _gst_251 = dq_base + qoff_13; - __builtin_assume(__isGlobal(_gst_251)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_251) + (0)) = (__nv_bfloat16)dq_o_13; + auto _gst_283 = dq_base + qoff_13; + __builtin_assume(__isGlobal(_gst_283)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_283) + (0)) = (__nv_bfloat16)dq_o_13; } { - auto _gst_252 = dk_base + qoff_13; - __builtin_assume(__isGlobal(_gst_252)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_252) + (0)) = (__nv_bfloat16)dk_o_13; + auto _gst_284 = dk_base + qoff_13; + __builtin_assume(__isGlobal(_gst_284)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dk_o_13; } } int t_14 = 48 + 8 * th + 6; @@ -3585,9 +3774,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_293[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_14 = __uint_as_float(r_293[0] << 16); float q_t0_14 = ((t_14 >= clen) ? 0.0f : q_t_14); - float dqf_14 = pool_f32[ew_0 + 768 + d_lane]; - float dkf_14 = pool_f32[ew_0 + 1792 + d_lane]; - float dgf_14 = pool_f32[ew_0 + 2816 + d_lane]; float bt_t_14 = btv_0[6]; float e_off_14 = approx_exp2(g_t_14 - gn_0); float e_dg_14 = approx_exp2(g_t_14 - gc_4); @@ -3595,22 +3781,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_14 = __fmaf_rn(ko_0[6], e_off_14, kdg_0[6] * e_dg_14); dbp_0[6] = dk2_14 * k_t_14; float dk2b_14 = dk2_14 * bt_t_14; + float q_r_14 = q_t0_14; + float qe_13 = q_r_14 * e_dg_14; + float kp_29 = k_t_14 * bt_t_14; + __nv_bfloat16 kb_13 = (__nv_bfloat16)kp_29; + float kbe_13 = (float)kb_13 * e_dg_14; + float cr_13 = __fmaf_rn(__uint_as_float(__as_u32(qe_13) & 4294959104u) - qe_13, qdg_0[6], (__uint_as_float(__as_u32(kbe_13) & 4294959104u) - kp_29 * e_dg_14) * kdg_0[6]); + float dqf_14 = pool_f32[ew_0 + 768 + d_lane]; + float dkf_14 = pool_f32[ew_0 + 1792 + d_lane]; + float dgf_14 = pool_f32[ew_0 + 2816 + d_lane]; float dq_o_14 = dq2_14 + dqf_14; float dk_o_14 = dk2b_14 + dkf_14 + dkt_4[6]; - float q_r_14 = q_t0_14; - float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, (dk2b_14 - dkt_4[6]) * k_t_14)) + dgf_14; + float dg_o_14 = __fmaf_rn(q_r_14, dqf_14, __fmaf_rn(q_r_14, dq2_14, __fmaf_rn(dk2b_14 - dkt_4[6], k_t_14, cr_13))) + dgf_14; dgo_0[6] = dg_o_14; int qoff_14 = t_14 * qrs; if (t_14 < clen) { { - auto _gst_253 = dq_base + qoff_14; - __builtin_assume(__isGlobal(_gst_253)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_253) + (0)) = (__nv_bfloat16)dq_o_14; + auto _gst_285 = dq_base + qoff_14; + __builtin_assume(__isGlobal(_gst_285)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dq_o_14; } { - auto _gst_254 = dk_base + qoff_14; - __builtin_assume(__isGlobal(_gst_254)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_254) + (0)) = (__nv_bfloat16)dk_o_14; + auto _gst_286 = dk_base + qoff_14; + __builtin_assume(__isGlobal(_gst_286)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dk_o_14; } } int t_15 = 48 + 8 * th + 7; @@ -3624,9 +3818,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_296[0])) : "r"((qbase_4 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_15 = __uint_as_float(r_296[0] << 16); float q_t0_15 = ((t_15 >= clen) ? 0.0f : q_t_15); - float dqf_15 = pool_f32[ew_0 + 896 + d_lane]; - float dkf_15 = pool_f32[ew_0 + 1920 + d_lane]; - float dgf_15 = pool_f32[ew_0 + 2944 + d_lane]; float bt_t_15 = btv_0[7]; float e_off_15 = approx_exp2(g_t_15 - gn_0); float e_dg_15 = approx_exp2(g_t_15 - gc_4); @@ -3634,22 +3825,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_15 = __fmaf_rn(ko_0[7], e_off_15, kdg_0[7] * e_dg_15); dbp_0[7] = dk2_15 * k_t_15; float dk2b_15 = dk2_15 * bt_t_15; + float q_r_15 = q_t0_15; + float qe_14 = q_r_15 * e_dg_15; + float kp_30 = k_t_15 * bt_t_15; + __nv_bfloat16 kb_14 = (__nv_bfloat16)kp_30; + float kbe_14 = (float)kb_14 * e_dg_15; + float cr_14 = __fmaf_rn(__uint_as_float(__as_u32(qe_14) & 4294959104u) - qe_14, qdg_0[7], (__uint_as_float(__as_u32(kbe_14) & 4294959104u) - kp_30 * e_dg_15) * kdg_0[7]); + float dqf_15 = pool_f32[ew_0 + 896 + d_lane]; + float dkf_15 = pool_f32[ew_0 + 1920 + d_lane]; + float dgf_15 = pool_f32[ew_0 + 2944 + d_lane]; float dq_o_15 = dq2_15 + dqf_15; float dk_o_15 = dk2b_15 + dkf_15 + dkt_4[7]; - float q_r_15 = q_t0_15; - float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, (dk2b_15 - dkt_4[7]) * k_t_15)) + dgf_15; + float dg_o_15 = __fmaf_rn(q_r_15, dqf_15, __fmaf_rn(q_r_15, dq2_15, __fmaf_rn(dk2b_15 - dkt_4[7], k_t_15, cr_14))) + dgf_15; dgo_0[7] = dg_o_15; int qoff_15 = t_15 * qrs; if (t_15 < clen) { { - auto _gst_255 = dq_base + qoff_15; - __builtin_assume(__isGlobal(_gst_255)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_255) + (0)) = (__nv_bfloat16)dq_o_15; + auto _gst_287 = dq_base + qoff_15; + __builtin_assume(__isGlobal(_gst_287)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dq_o_15; } { - auto _gst_256 = dk_base + qoff_15; - __builtin_assume(__isGlobal(_gst_256)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_256) + (0)) = (__nv_bfloat16)dk_o_15; + auto _gst_288 = dk_base + qoff_15; + __builtin_assume(__isGlobal(_gst_288)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dk_o_15; } } float s_run_0[1]; @@ -3678,36 +3877,36 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int b2_0 = lane_1 >> 2 & 1; float v4_0[4]; float send_6 = ((b4_0 == 0) ? dbp_0[4] : dbp_0[0]); - float _shfl_xor_sync_257 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); - v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_257; + float _shfl_xor_sync_289 = __shfl_xor_sync(0xFFFFFFFF, send_6, 16); + v4_0[0] = ((b4_0 == 0) ? dbp_0[0] : dbp_0[4]) + _shfl_xor_sync_289; float send_7 = ((b4_0 == 0) ? dbp_0[5] : dbp_0[1]); - float _shfl_xor_sync_258 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); - v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_258; + float _shfl_xor_sync_290 = __shfl_xor_sync(0xFFFFFFFF, send_7, 16); + v4_0[1] = ((b4_0 == 0) ? dbp_0[1] : dbp_0[5]) + _shfl_xor_sync_290; float send_8 = ((b4_0 == 0) ? dbp_0[6] : dbp_0[2]); - float _shfl_xor_sync_259 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); - v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_259; + float _shfl_xor_sync_291 = __shfl_xor_sync(0xFFFFFFFF, send_8, 16); + v4_0[2] = ((b4_0 == 0) ? dbp_0[2] : dbp_0[6]) + _shfl_xor_sync_291; float send_9 = ((b4_0 == 0) ? dbp_0[7] : dbp_0[3]); - float _shfl_xor_sync_260 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); - v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_260; + float _shfl_xor_sync_292 = __shfl_xor_sync(0xFFFFFFFF, send_9, 16); + v4_0[3] = ((b4_0 == 0) ? dbp_0[3] : dbp_0[7]) + _shfl_xor_sync_292; float v2_24[2]; float send_10 = ((b3_0 == 0) ? v4_0[2] : v4_0[0]); - float _shfl_xor_sync_261 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); - v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_261; + float _shfl_xor_sync_293 = __shfl_xor_sync(0xFFFFFFFF, send_10, 8); + v2_24[0] = ((b3_0 == 0) ? v4_0[0] : v4_0[2]) + _shfl_xor_sync_293; float send_11 = ((b3_0 == 0) ? v4_0[3] : v4_0[1]); - float _shfl_xor_sync_262 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); - v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_262; + float _shfl_xor_sync_294 = __shfl_xor_sync(0xFFFFFFFF, send_11, 8); + v2_24[1] = ((b3_0 == 0) ? v4_0[1] : v4_0[3]) + _shfl_xor_sync_294; float send_12 = ((b2_0 == 0) ? v2_24[1] : v2_24[0]); - float _shfl_xor_sync_263 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); - float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_263; - float _shfl_xor_sync_264 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); - v1_0 = v1_0 + _shfl_xor_sync_264; - float _shfl_xor_sync_265 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); - v1_0 = v1_0 + _shfl_xor_sync_265; + float _shfl_xor_sync_295 = __shfl_xor_sync(0xFFFFFFFF, send_12, 4); + float v1_0 = ((b2_0 == 0) ? v2_24[0] : v2_24[1]) + _shfl_xor_sync_295; + float _shfl_xor_sync_296 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 2); + v1_0 = v1_0 + _shfl_xor_sync_296; + float _shfl_xor_sync_297 = __shfl_xor_sync(0xFFFFFFFF, v1_0, 1); + v1_0 = v1_0 + _shfl_xor_sync_297; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (th) * 8); { - uint32_t _addr_266 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_266), "f"(tot_0) : "memory"); + uint32_t _addr_298 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_298), "f"(tot_0) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -3725,9 +3924,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_0 = offset + sfx[0]; if (t_16 < clen) { { - auto _gst_267 = dg_base + goff_0; - __builtin_assume(__isGlobal(_gst_267)); - *(reinterpret_cast(_gst_267) + (0)) = dg_0; + auto _gst_299 = dg_base + goff_0; + __builtin_assume(__isGlobal(_gst_299)); + *(reinterpret_cast(_gst_299) + (0)) = dg_0; } } int t_17 = 48 + 8 * th + 1; @@ -3735,9 +3934,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_1 = offset + sfx[1]; if (t_17 < clen) { { - auto _gst_268 = dg_base + goff_1; - __builtin_assume(__isGlobal(_gst_268)); - *(reinterpret_cast(_gst_268) + (0)) = dg_1; + auto _gst_300 = dg_base + goff_1; + __builtin_assume(__isGlobal(_gst_300)); + *(reinterpret_cast(_gst_300) + (0)) = dg_1; } } int t_18 = 48 + 8 * th + 2; @@ -3745,9 +3944,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_2 = offset + sfx[2]; if (t_18 < clen) { { - auto _gst_269 = dg_base + goff_2; - __builtin_assume(__isGlobal(_gst_269)); - *(reinterpret_cast(_gst_269) + (0)) = dg_2; + auto _gst_301 = dg_base + goff_2; + __builtin_assume(__isGlobal(_gst_301)); + *(reinterpret_cast(_gst_301) + (0)) = dg_2; } } int t_19 = 48 + 8 * th + 3; @@ -3755,9 +3954,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_3 = offset + sfx[3]; if (t_19 < clen) { { - auto _gst_270 = dg_base + goff_3; - __builtin_assume(__isGlobal(_gst_270)); - *(reinterpret_cast(_gst_270) + (0)) = dg_3; + auto _gst_302 = dg_base + goff_3; + __builtin_assume(__isGlobal(_gst_302)); + *(reinterpret_cast(_gst_302) + (0)) = dg_3; } } int t_20 = 48 + 8 * th + 4; @@ -3765,9 +3964,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_4 = offset + sfx[4]; if (t_20 < clen) { { - auto _gst_271 = dg_base + goff_4; - __builtin_assume(__isGlobal(_gst_271)); - *(reinterpret_cast(_gst_271) + (0)) = dg_4; + auto _gst_303 = dg_base + goff_4; + __builtin_assume(__isGlobal(_gst_303)); + *(reinterpret_cast(_gst_303) + (0)) = dg_4; } } int t_21 = 48 + 8 * th + 5; @@ -3775,9 +3974,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_5 = offset + sfx[5]; if (t_21 < clen) { { - auto _gst_272 = dg_base + goff_5; - __builtin_assume(__isGlobal(_gst_272)); - *(reinterpret_cast(_gst_272) + (0)) = dg_5; + auto _gst_304 = dg_base + goff_5; + __builtin_assume(__isGlobal(_gst_304)); + *(reinterpret_cast(_gst_304) + (0)) = dg_5; } } int t_22 = 48 + 8 * th + 6; @@ -3785,9 +3984,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_6 = offset + sfx[6]; if (t_22 < clen) { { - auto _gst_273 = dg_base + goff_6; - __builtin_assume(__isGlobal(_gst_273)); - *(reinterpret_cast(_gst_273) + (0)) = dg_6; + auto _gst_305 = dg_base + goff_6; + __builtin_assume(__isGlobal(_gst_305)); + *(reinterpret_cast(_gst_305) + (0)) = dg_6; } } int t_23 = 48 + 8 * th + 7; @@ -3795,9 +3994,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_7 = offset + sfx[7]; if (t_23 < clen) { { - auto _gst_274 = dg_base + goff_7; - __builtin_assume(__isGlobal(_gst_274)); - *(reinterpret_cast(_gst_274) + (0)) = dg_7; + auto _gst_306 = dg_base + goff_7; + __builtin_assume(__isGlobal(_gst_306)); + *(reinterpret_cast(_gst_306) + (0)) = dg_7; } } off_acc[0] = off_acc[0] + t_hi + t_lo; @@ -3815,9 +4014,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_8 = offset_0 + sfx_0[0]; if (t_24 < clen) { { - auto _gst_275 = dg_base + goff_8; - __builtin_assume(__isGlobal(_gst_275)); - *(reinterpret_cast(_gst_275) + (0)) = dg_8; + auto _gst_307 = dg_base + goff_8; + __builtin_assume(__isGlobal(_gst_307)); + *(reinterpret_cast(_gst_307) + (0)) = dg_8; } } int t_25 = 32 + 8 * th + 1; @@ -3825,9 +4024,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_9 = offset_0 + sfx_0[1]; if (t_25 < clen) { { - auto _gst_276 = dg_base + goff_9; - __builtin_assume(__isGlobal(_gst_276)); - *(reinterpret_cast(_gst_276) + (0)) = dg_9; + auto _gst_308 = dg_base + goff_9; + __builtin_assume(__isGlobal(_gst_308)); + *(reinterpret_cast(_gst_308) + (0)) = dg_9; } } int t_26 = 32 + 8 * th + 2; @@ -3835,9 +4034,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_10 = offset_0 + sfx_0[2]; if (t_26 < clen) { { - auto _gst_277 = dg_base + goff_10; - __builtin_assume(__isGlobal(_gst_277)); - *(reinterpret_cast(_gst_277) + (0)) = dg_10; + auto _gst_309 = dg_base + goff_10; + __builtin_assume(__isGlobal(_gst_309)); + *(reinterpret_cast(_gst_309) + (0)) = dg_10; } } int t_27 = 32 + 8 * th + 3; @@ -3845,9 +4044,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_11 = offset_0 + sfx_0[3]; if (t_27 < clen) { { - auto _gst_278 = dg_base + goff_11; - __builtin_assume(__isGlobal(_gst_278)); - *(reinterpret_cast(_gst_278) + (0)) = dg_11; + auto _gst_310 = dg_base + goff_11; + __builtin_assume(__isGlobal(_gst_310)); + *(reinterpret_cast(_gst_310) + (0)) = dg_11; } } int t_28 = 32 + 8 * th + 4; @@ -3855,9 +4054,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_12 = offset_0 + sfx_0[4]; if (t_28 < clen) { { - auto _gst_279 = dg_base + goff_12; - __builtin_assume(__isGlobal(_gst_279)); - *(reinterpret_cast(_gst_279) + (0)) = dg_12; + auto _gst_311 = dg_base + goff_12; + __builtin_assume(__isGlobal(_gst_311)); + *(reinterpret_cast(_gst_311) + (0)) = dg_12; } } int t_29 = 32 + 8 * th + 5; @@ -3865,9 +4064,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_13 = offset_0 + sfx_0[5]; if (t_29 < clen) { { - auto _gst_280 = dg_base + goff_13; - __builtin_assume(__isGlobal(_gst_280)); - *(reinterpret_cast(_gst_280) + (0)) = dg_13; + auto _gst_312 = dg_base + goff_13; + __builtin_assume(__isGlobal(_gst_312)); + *(reinterpret_cast(_gst_312) + (0)) = dg_13; } } int t_30 = 32 + 8 * th + 6; @@ -3875,9 +4074,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_14 = offset_0 + sfx_0[6]; if (t_30 < clen) { { - auto _gst_281 = dg_base + goff_14; - __builtin_assume(__isGlobal(_gst_281)); - *(reinterpret_cast(_gst_281) + (0)) = dg_14; + auto _gst_313 = dg_base + goff_14; + __builtin_assume(__isGlobal(_gst_313)); + *(reinterpret_cast(_gst_313) + (0)) = dg_14; } } int t_31 = 32 + 8 * th + 7; @@ -3885,15 +4084,15 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_15 = offset_0 + sfx_0[7]; if (t_31 < clen) { { - auto _gst_282 = dg_base + goff_15; - __builtin_assume(__isGlobal(_gst_282)); - *(reinterpret_cast(_gst_282) + (0)) = dg_15; + auto _gst_314 = dg_base + goff_15; + __builtin_assume(__isGlobal(_gst_314)); + *(reinterpret_cast(_gst_314) + (0)) = dg_15; } } off_acc[0] = off_acc[0] + t_hi_0 + t_lo_0; { - uint32_t _addr_283 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_283), "f"(off_acc[0]) : "memory"); + uint32_t _addr_315 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_315), "f"(off_acc[0]) : "memory"); } mbarrier_wait(p1b_ready_addr, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -3940,9 +4139,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_300[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_16 = __uint_as_float(r_300[0] << 16); float q_t0_16 = ((t_32 >= clen) ? 0.0f : q_t_16); - float dqf_16 = pool_f32[ew_1 + d_lane]; - float dkf_16 = pool_f32[ew_1 + 1024 + d_lane]; - float dgf_16 = pool_f32[ew_1 + 2048 + d_lane]; float bt_t_16 = btv_1[0]; float e_off_16 = approx_exp2(g_t_16 - gn_1); float e_dg_16 = approx_exp2(g_t_16 - gc_5); @@ -3950,22 +4146,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_16 = __fmaf_rn(ko_1[0], e_off_16, kdg_1[0] * e_dg_16); dbp_1[0] = dk2_16 * k_t_16; float dk2b_16 = dk2_16 * bt_t_16; + float q_r_16 = q_t0_16; + float qe_15 = q_r_16 * e_dg_16; + float kp_31 = k_t_16 * bt_t_16; + __nv_bfloat16 kb_15 = (__nv_bfloat16)kp_31; + float kbe_15 = (float)kb_15 * e_dg_16; + float cr_15 = __fmaf_rn(__uint_as_float(__as_u32(qe_15) & 4294959104u) - qe_15, qdg_1[0], (__uint_as_float(__as_u32(kbe_15) & 4294959104u) - kp_31 * e_dg_16) * kdg_1[0]); + float dqf_16 = pool_f32[ew_1 + d_lane]; + float dkf_16 = pool_f32[ew_1 + 1024 + d_lane]; + float dgf_16 = pool_f32[ew_1 + 2048 + d_lane]; float dq_o_16 = dq2_16 + dqf_16; float dk_o_16 = dk2b_16 + dkf_16 + dkt_5[0]; - float q_r_16 = q_t0_16; - float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, (dk2b_16 - dkt_5[0]) * k_t_16)) + dgf_16; + float dg_o_16 = __fmaf_rn(q_r_16, dqf_16, __fmaf_rn(q_r_16, dq2_16, __fmaf_rn(dk2b_16 - dkt_5[0], k_t_16, cr_15))) + dgf_16; dgo_1[0] = dg_o_16; int qoff_16 = t_32 * qrs; if (t_32 < clen) { { - auto _gst_284 = dq_base + qoff_16; - __builtin_assume(__isGlobal(_gst_284)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_284) + (0)) = (__nv_bfloat16)dq_o_16; + auto _gst_316 = dq_base + qoff_16; + __builtin_assume(__isGlobal(_gst_316)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_316) + (0)) = (__nv_bfloat16)dq_o_16; } { - auto _gst_285 = dk_base + qoff_16; - __builtin_assume(__isGlobal(_gst_285)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_285) + (0)) = (__nv_bfloat16)dk_o_16; + auto _gst_317 = dk_base + qoff_16; + __builtin_assume(__isGlobal(_gst_317)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_317) + (0)) = (__nv_bfloat16)dk_o_16; } } int t_33 = 16 + 8 * th + 1; @@ -3979,9 +4183,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_303[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_17 = __uint_as_float(r_303[0] << 16); float q_t0_17 = ((t_33 >= clen) ? 0.0f : q_t_17); - float dqf_17 = pool_f32[ew_1 + 128 + d_lane]; - float dkf_17 = pool_f32[ew_1 + 1152 + d_lane]; - float dgf_17 = pool_f32[ew_1 + 2176 + d_lane]; float bt_t_17 = btv_1[1]; float e_off_17 = approx_exp2(g_t_17 - gn_1); float e_dg_17 = approx_exp2(g_t_17 - gc_5); @@ -3989,22 +4190,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_17 = __fmaf_rn(ko_1[1], e_off_17, kdg_1[1] * e_dg_17); dbp_1[1] = dk2_17 * k_t_17; float dk2b_17 = dk2_17 * bt_t_17; + float q_r_17 = q_t0_17; + float qe_16 = q_r_17 * e_dg_17; + float kp_32 = k_t_17 * bt_t_17; + __nv_bfloat16 kb_16 = (__nv_bfloat16)kp_32; + float kbe_16 = (float)kb_16 * e_dg_17; + float cr_16 = __fmaf_rn(__uint_as_float(__as_u32(qe_16) & 4294959104u) - qe_16, qdg_1[1], (__uint_as_float(__as_u32(kbe_16) & 4294959104u) - kp_32 * e_dg_17) * kdg_1[1]); + float dqf_17 = pool_f32[ew_1 + 128 + d_lane]; + float dkf_17 = pool_f32[ew_1 + 1152 + d_lane]; + float dgf_17 = pool_f32[ew_1 + 2176 + d_lane]; float dq_o_17 = dq2_17 + dqf_17; float dk_o_17 = dk2b_17 + dkf_17 + dkt_5[1]; - float q_r_17 = q_t0_17; - float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, (dk2b_17 - dkt_5[1]) * k_t_17)) + dgf_17; + float dg_o_17 = __fmaf_rn(q_r_17, dqf_17, __fmaf_rn(q_r_17, dq2_17, __fmaf_rn(dk2b_17 - dkt_5[1], k_t_17, cr_16))) + dgf_17; dgo_1[1] = dg_o_17; int qoff_17 = t_33 * qrs; if (t_33 < clen) { { - auto _gst_286 = dq_base + qoff_17; - __builtin_assume(__isGlobal(_gst_286)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_286) + (0)) = (__nv_bfloat16)dq_o_17; + auto _gst_318 = dq_base + qoff_17; + __builtin_assume(__isGlobal(_gst_318)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_318) + (0)) = (__nv_bfloat16)dq_o_17; } { - auto _gst_287 = dk_base + qoff_17; - __builtin_assume(__isGlobal(_gst_287)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_287) + (0)) = (__nv_bfloat16)dk_o_17; + auto _gst_319 = dk_base + qoff_17; + __builtin_assume(__isGlobal(_gst_319)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dk_o_17; } } int t_34 = 16 + 8 * th + 2; @@ -4018,9 +4227,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_306[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_18 = __uint_as_float(r_306[0] << 16); float q_t0_18 = ((t_34 >= clen) ? 0.0f : q_t_18); - float dqf_18 = pool_f32[ew_1 + 256 + d_lane]; - float dkf_18 = pool_f32[ew_1 + 1280 + d_lane]; - float dgf_18 = pool_f32[ew_1 + 2304 + d_lane]; float bt_t_18 = btv_1[2]; float e_off_18 = approx_exp2(g_t_18 - gn_1); float e_dg_18 = approx_exp2(g_t_18 - gc_5); @@ -4028,22 +4234,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_18 = __fmaf_rn(ko_1[2], e_off_18, kdg_1[2] * e_dg_18); dbp_1[2] = dk2_18 * k_t_18; float dk2b_18 = dk2_18 * bt_t_18; + float q_r_18 = q_t0_18; + float qe_17 = q_r_18 * e_dg_18; + float kp_33 = k_t_18 * bt_t_18; + __nv_bfloat16 kb_17 = (__nv_bfloat16)kp_33; + float kbe_17 = (float)kb_17 * e_dg_18; + float cr_17 = __fmaf_rn(__uint_as_float(__as_u32(qe_17) & 4294959104u) - qe_17, qdg_1[2], (__uint_as_float(__as_u32(kbe_17) & 4294959104u) - kp_33 * e_dg_18) * kdg_1[2]); + float dqf_18 = pool_f32[ew_1 + 256 + d_lane]; + float dkf_18 = pool_f32[ew_1 + 1280 + d_lane]; + float dgf_18 = pool_f32[ew_1 + 2304 + d_lane]; float dq_o_18 = dq2_18 + dqf_18; float dk_o_18 = dk2b_18 + dkf_18 + dkt_5[2]; - float q_r_18 = q_t0_18; - float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, (dk2b_18 - dkt_5[2]) * k_t_18)) + dgf_18; + float dg_o_18 = __fmaf_rn(q_r_18, dqf_18, __fmaf_rn(q_r_18, dq2_18, __fmaf_rn(dk2b_18 - dkt_5[2], k_t_18, cr_17))) + dgf_18; dgo_1[2] = dg_o_18; int qoff_18 = t_34 * qrs; if (t_34 < clen) { { - auto _gst_288 = dq_base + qoff_18; - __builtin_assume(__isGlobal(_gst_288)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_288) + (0)) = (__nv_bfloat16)dq_o_18; + auto _gst_320 = dq_base + qoff_18; + __builtin_assume(__isGlobal(_gst_320)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dq_o_18; } { - auto _gst_289 = dk_base + qoff_18; - __builtin_assume(__isGlobal(_gst_289)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_289) + (0)) = (__nv_bfloat16)dk_o_18; + auto _gst_321 = dk_base + qoff_18; + __builtin_assume(__isGlobal(_gst_321)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dk_o_18; } } int t_35 = 16 + 8 * th + 3; @@ -4057,9 +4271,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_309[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_19 = __uint_as_float(r_309[0] << 16); float q_t0_19 = ((t_35 >= clen) ? 0.0f : q_t_19); - float dqf_19 = pool_f32[ew_1 + 384 + d_lane]; - float dkf_19 = pool_f32[ew_1 + 1408 + d_lane]; - float dgf_19 = pool_f32[ew_1 + 2432 + d_lane]; float bt_t_19 = btv_1[3]; float e_off_19 = approx_exp2(g_t_19 - gn_1); float e_dg_19 = approx_exp2(g_t_19 - gc_5); @@ -4067,22 +4278,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_19 = __fmaf_rn(ko_1[3], e_off_19, kdg_1[3] * e_dg_19); dbp_1[3] = dk2_19 * k_t_19; float dk2b_19 = dk2_19 * bt_t_19; + float q_r_19 = q_t0_19; + float qe_18 = q_r_19 * e_dg_19; + float kp_34 = k_t_19 * bt_t_19; + __nv_bfloat16 kb_18 = (__nv_bfloat16)kp_34; + float kbe_18 = (float)kb_18 * e_dg_19; + float cr_18 = __fmaf_rn(__uint_as_float(__as_u32(qe_18) & 4294959104u) - qe_18, qdg_1[3], (__uint_as_float(__as_u32(kbe_18) & 4294959104u) - kp_34 * e_dg_19) * kdg_1[3]); + float dqf_19 = pool_f32[ew_1 + 384 + d_lane]; + float dkf_19 = pool_f32[ew_1 + 1408 + d_lane]; + float dgf_19 = pool_f32[ew_1 + 2432 + d_lane]; float dq_o_19 = dq2_19 + dqf_19; float dk_o_19 = dk2b_19 + dkf_19 + dkt_5[3]; - float q_r_19 = q_t0_19; - float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, (dk2b_19 - dkt_5[3]) * k_t_19)) + dgf_19; + float dg_o_19 = __fmaf_rn(q_r_19, dqf_19, __fmaf_rn(q_r_19, dq2_19, __fmaf_rn(dk2b_19 - dkt_5[3], k_t_19, cr_18))) + dgf_19; dgo_1[3] = dg_o_19; int qoff_19 = t_35 * qrs; if (t_35 < clen) { { - auto _gst_290 = dq_base + qoff_19; - __builtin_assume(__isGlobal(_gst_290)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_290) + (0)) = (__nv_bfloat16)dq_o_19; + auto _gst_322 = dq_base + qoff_19; + __builtin_assume(__isGlobal(_gst_322)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dq_o_19; } { - auto _gst_291 = dk_base + qoff_19; - __builtin_assume(__isGlobal(_gst_291)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_291) + (0)) = (__nv_bfloat16)dk_o_19; + auto _gst_323 = dk_base + qoff_19; + __builtin_assume(__isGlobal(_gst_323)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dk_o_19; } } int t_36 = 16 + 8 * th + 4; @@ -4096,9 +4315,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_312[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_20 = __uint_as_float(r_312[0] << 16); float q_t0_20 = ((t_36 >= clen) ? 0.0f : q_t_20); - float dqf_20 = pool_f32[ew_1 + 512 + d_lane]; - float dkf_20 = pool_f32[ew_1 + 1536 + d_lane]; - float dgf_20 = pool_f32[ew_1 + 2560 + d_lane]; float bt_t_20 = btv_1[4]; float e_off_20 = approx_exp2(g_t_20 - gn_1); float e_dg_20 = approx_exp2(g_t_20 - gc_5); @@ -4106,22 +4322,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_20 = __fmaf_rn(ko_1[4], e_off_20, kdg_1[4] * e_dg_20); dbp_1[4] = dk2_20 * k_t_20; float dk2b_20 = dk2_20 * bt_t_20; + float q_r_20 = q_t0_20; + float qe_19 = q_r_20 * e_dg_20; + float kp_35 = k_t_20 * bt_t_20; + __nv_bfloat16 kb_19 = (__nv_bfloat16)kp_35; + float kbe_19 = (float)kb_19 * e_dg_20; + float cr_19 = __fmaf_rn(__uint_as_float(__as_u32(qe_19) & 4294959104u) - qe_19, qdg_1[4], (__uint_as_float(__as_u32(kbe_19) & 4294959104u) - kp_35 * e_dg_20) * kdg_1[4]); + float dqf_20 = pool_f32[ew_1 + 512 + d_lane]; + float dkf_20 = pool_f32[ew_1 + 1536 + d_lane]; + float dgf_20 = pool_f32[ew_1 + 2560 + d_lane]; float dq_o_20 = dq2_20 + dqf_20; float dk_o_20 = dk2b_20 + dkf_20 + dkt_5[4]; - float q_r_20 = q_t0_20; - float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, (dk2b_20 - dkt_5[4]) * k_t_20)) + dgf_20; + float dg_o_20 = __fmaf_rn(q_r_20, dqf_20, __fmaf_rn(q_r_20, dq2_20, __fmaf_rn(dk2b_20 - dkt_5[4], k_t_20, cr_19))) + dgf_20; dgo_1[4] = dg_o_20; int qoff_20 = t_36 * qrs; if (t_36 < clen) { { - auto _gst_292 = dq_base + qoff_20; - __builtin_assume(__isGlobal(_gst_292)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_292) + (0)) = (__nv_bfloat16)dq_o_20; + auto _gst_324 = dq_base + qoff_20; + __builtin_assume(__isGlobal(_gst_324)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dq_o_20; } { - auto _gst_293 = dk_base + qoff_20; - __builtin_assume(__isGlobal(_gst_293)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_293) + (0)) = (__nv_bfloat16)dk_o_20; + auto _gst_325 = dk_base + qoff_20; + __builtin_assume(__isGlobal(_gst_325)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dk_o_20; } } int t_37 = 16 + 8 * th + 5; @@ -4135,9 +4359,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_315[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_21 = __uint_as_float(r_315[0] << 16); float q_t0_21 = ((t_37 >= clen) ? 0.0f : q_t_21); - float dqf_21 = pool_f32[ew_1 + 640 + d_lane]; - float dkf_21 = pool_f32[ew_1 + 1664 + d_lane]; - float dgf_21 = pool_f32[ew_1 + 2688 + d_lane]; float bt_t_21 = btv_1[5]; float e_off_21 = approx_exp2(g_t_21 - gn_1); float e_dg_21 = approx_exp2(g_t_21 - gc_5); @@ -4145,22 +4366,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_21 = __fmaf_rn(ko_1[5], e_off_21, kdg_1[5] * e_dg_21); dbp_1[5] = dk2_21 * k_t_21; float dk2b_21 = dk2_21 * bt_t_21; + float q_r_21 = q_t0_21; + float qe_20 = q_r_21 * e_dg_21; + float kp_36 = k_t_21 * bt_t_21; + __nv_bfloat16 kb_20 = (__nv_bfloat16)kp_36; + float kbe_20 = (float)kb_20 * e_dg_21; + float cr_20 = __fmaf_rn(__uint_as_float(__as_u32(qe_20) & 4294959104u) - qe_20, qdg_1[5], (__uint_as_float(__as_u32(kbe_20) & 4294959104u) - kp_36 * e_dg_21) * kdg_1[5]); + float dqf_21 = pool_f32[ew_1 + 640 + d_lane]; + float dkf_21 = pool_f32[ew_1 + 1664 + d_lane]; + float dgf_21 = pool_f32[ew_1 + 2688 + d_lane]; float dq_o_21 = dq2_21 + dqf_21; float dk_o_21 = dk2b_21 + dkf_21 + dkt_5[5]; - float q_r_21 = q_t0_21; - float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, (dk2b_21 - dkt_5[5]) * k_t_21)) + dgf_21; + float dg_o_21 = __fmaf_rn(q_r_21, dqf_21, __fmaf_rn(q_r_21, dq2_21, __fmaf_rn(dk2b_21 - dkt_5[5], k_t_21, cr_20))) + dgf_21; dgo_1[5] = dg_o_21; int qoff_21 = t_37 * qrs; if (t_37 < clen) { { - auto _gst_294 = dq_base + qoff_21; - __builtin_assume(__isGlobal(_gst_294)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_294) + (0)) = (__nv_bfloat16)dq_o_21; + auto _gst_326 = dq_base + qoff_21; + __builtin_assume(__isGlobal(_gst_326)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dq_o_21; } { - auto _gst_295 = dk_base + qoff_21; - __builtin_assume(__isGlobal(_gst_295)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_295) + (0)) = (__nv_bfloat16)dk_o_21; + auto _gst_327 = dk_base + qoff_21; + __builtin_assume(__isGlobal(_gst_327)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dk_o_21; } } int t_38 = 16 + 8 * th + 6; @@ -4174,9 +4403,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_318[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_22 = __uint_as_float(r_318[0] << 16); float q_t0_22 = ((t_38 >= clen) ? 0.0f : q_t_22); - float dqf_22 = pool_f32[ew_1 + 768 + d_lane]; - float dkf_22 = pool_f32[ew_1 + 1792 + d_lane]; - float dgf_22 = pool_f32[ew_1 + 2816 + d_lane]; float bt_t_22 = btv_1[6]; float e_off_22 = approx_exp2(g_t_22 - gn_1); float e_dg_22 = approx_exp2(g_t_22 - gc_5); @@ -4184,22 +4410,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_22 = __fmaf_rn(ko_1[6], e_off_22, kdg_1[6] * e_dg_22); dbp_1[6] = dk2_22 * k_t_22; float dk2b_22 = dk2_22 * bt_t_22; + float q_r_22 = q_t0_22; + float qe_21 = q_r_22 * e_dg_22; + float kp_37 = k_t_22 * bt_t_22; + __nv_bfloat16 kb_21 = (__nv_bfloat16)kp_37; + float kbe_21 = (float)kb_21 * e_dg_22; + float cr_21 = __fmaf_rn(__uint_as_float(__as_u32(qe_21) & 4294959104u) - qe_21, qdg_1[6], (__uint_as_float(__as_u32(kbe_21) & 4294959104u) - kp_37 * e_dg_22) * kdg_1[6]); + float dqf_22 = pool_f32[ew_1 + 768 + d_lane]; + float dkf_22 = pool_f32[ew_1 + 1792 + d_lane]; + float dgf_22 = pool_f32[ew_1 + 2816 + d_lane]; float dq_o_22 = dq2_22 + dqf_22; float dk_o_22 = dk2b_22 + dkf_22 + dkt_5[6]; - float q_r_22 = q_t0_22; - float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, (dk2b_22 - dkt_5[6]) * k_t_22)) + dgf_22; + float dg_o_22 = __fmaf_rn(q_r_22, dqf_22, __fmaf_rn(q_r_22, dq2_22, __fmaf_rn(dk2b_22 - dkt_5[6], k_t_22, cr_21))) + dgf_22; dgo_1[6] = dg_o_22; int qoff_22 = t_38 * qrs; if (t_38 < clen) { { - auto _gst_296 = dq_base + qoff_22; - __builtin_assume(__isGlobal(_gst_296)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_296) + (0)) = (__nv_bfloat16)dq_o_22; + auto _gst_328 = dq_base + qoff_22; + __builtin_assume(__isGlobal(_gst_328)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dq_o_22; } { - auto _gst_297 = dk_base + qoff_22; - __builtin_assume(__isGlobal(_gst_297)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_297) + (0)) = (__nv_bfloat16)dk_o_22; + auto _gst_329 = dk_base + qoff_22; + __builtin_assume(__isGlobal(_gst_329)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dk_o_22; } } int t_39 = 16 + 8 * th + 7; @@ -4213,9 +4447,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_321[0])) : "r"((qbase_5 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_23 = __uint_as_float(r_321[0] << 16); float q_t0_23 = ((t_39 >= clen) ? 0.0f : q_t_23); - float dqf_23 = pool_f32[ew_1 + 896 + d_lane]; - float dkf_23 = pool_f32[ew_1 + 1920 + d_lane]; - float dgf_23 = pool_f32[ew_1 + 2944 + d_lane]; float bt_t_23 = btv_1[7]; float e_off_23 = approx_exp2(g_t_23 - gn_1); float e_dg_23 = approx_exp2(g_t_23 - gc_5); @@ -4223,22 +4454,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_23 = __fmaf_rn(ko_1[7], e_off_23, kdg_1[7] * e_dg_23); dbp_1[7] = dk2_23 * k_t_23; float dk2b_23 = dk2_23 * bt_t_23; + float q_r_23 = q_t0_23; + float qe_22 = q_r_23 * e_dg_23; + float kp_38 = k_t_23 * bt_t_23; + __nv_bfloat16 kb_22 = (__nv_bfloat16)kp_38; + float kbe_22 = (float)kb_22 * e_dg_23; + float cr_22 = __fmaf_rn(__uint_as_float(__as_u32(qe_22) & 4294959104u) - qe_22, qdg_1[7], (__uint_as_float(__as_u32(kbe_22) & 4294959104u) - kp_38 * e_dg_23) * kdg_1[7]); + float dqf_23 = pool_f32[ew_1 + 896 + d_lane]; + float dkf_23 = pool_f32[ew_1 + 1920 + d_lane]; + float dgf_23 = pool_f32[ew_1 + 2944 + d_lane]; float dq_o_23 = dq2_23 + dqf_23; float dk_o_23 = dk2b_23 + dkf_23 + dkt_5[7]; - float q_r_23 = q_t0_23; - float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, (dk2b_23 - dkt_5[7]) * k_t_23)) + dgf_23; + float dg_o_23 = __fmaf_rn(q_r_23, dqf_23, __fmaf_rn(q_r_23, dq2_23, __fmaf_rn(dk2b_23 - dkt_5[7], k_t_23, cr_22))) + dgf_23; dgo_1[7] = dg_o_23; int qoff_23 = t_39 * qrs; if (t_39 < clen) { { - auto _gst_298 = dq_base + qoff_23; - __builtin_assume(__isGlobal(_gst_298)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_298) + (0)) = (__nv_bfloat16)dq_o_23; + auto _gst_330 = dq_base + qoff_23; + __builtin_assume(__isGlobal(_gst_330)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dq_o_23; } { - auto _gst_299 = dk_base + qoff_23; - __builtin_assume(__isGlobal(_gst_299)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_299) + (0)) = (__nv_bfloat16)dk_o_23; + auto _gst_331 = dk_base + qoff_23; + __builtin_assume(__isGlobal(_gst_331)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dk_o_23; } } float s_run_1[1]; @@ -4267,37 +4506,37 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int b2_1 = lane_1 >> 2 & 1; float v4_1[4]; float send_13 = ((b4_1 == 0) ? dbp_1[4] : dbp_1[0]); - float _shfl_xor_sync_300 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); - v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_300; + float _shfl_xor_sync_332 = __shfl_xor_sync(0xFFFFFFFF, send_13, 16); + v4_1[0] = ((b4_1 == 0) ? dbp_1[0] : dbp_1[4]) + _shfl_xor_sync_332; float send_14 = ((b4_1 == 0) ? dbp_1[5] : dbp_1[1]); - float _shfl_xor_sync_301 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); - v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_301; + float _shfl_xor_sync_333 = __shfl_xor_sync(0xFFFFFFFF, send_14, 16); + v4_1[1] = ((b4_1 == 0) ? dbp_1[1] : dbp_1[5]) + _shfl_xor_sync_333; float send_15 = ((b4_1 == 0) ? dbp_1[6] : dbp_1[2]); - float _shfl_xor_sync_302 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); - v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_302; + float _shfl_xor_sync_334 = __shfl_xor_sync(0xFFFFFFFF, send_15, 16); + v4_1[2] = ((b4_1 == 0) ? dbp_1[2] : dbp_1[6]) + _shfl_xor_sync_334; float send_16 = ((b4_1 == 0) ? dbp_1[7] : dbp_1[3]); - float _shfl_xor_sync_303 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); - v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_303; + float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_16, 16); + v4_1[3] = ((b4_1 == 0) ? dbp_1[3] : dbp_1[7]) + _shfl_xor_sync_335; float v2_25[2]; float send_17 = ((b3_1 == 0) ? v4_1[2] : v4_1[0]); - float _shfl_xor_sync_304 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); - v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_304; + float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_17, 8); + v2_25[0] = ((b3_1 == 0) ? v4_1[0] : v4_1[2]) + _shfl_xor_sync_336; float send_18 = ((b3_1 == 0) ? v4_1[3] : v4_1[1]); - float _shfl_xor_sync_305 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); - v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_305; + float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_18, 8); + v2_25[1] = ((b3_1 == 0) ? v4_1[1] : v4_1[3]) + _shfl_xor_sync_337; float send_19 = ((b2_1 == 0) ? v2_25[1] : v2_25[0]); - float _shfl_xor_sync_306 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); - float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_306; - float _shfl_xor_sync_307 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); - v1_1 = v1_1 + _shfl_xor_sync_307; - float _shfl_xor_sync_308 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); - v1_1 = v1_1 + _shfl_xor_sync_308; + float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_19, 4); + float v1_1 = ((b2_1 == 0) ? v2_25[0] : v2_25[1]) + _shfl_xor_sync_338; + float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 2); + v1_1 = v1_1 + _shfl_xor_sync_339; + float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, v1_1, 1); + v1_1 = v1_1 + _shfl_xor_sync_340; asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(epi_empty_addr + (2 + th) * 8); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_309 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_309), "f"(tot_1) : "memory"); + uint32_t _addr_341 = static_cast(pool_f32_addr + (unsigned int)(112640 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_341), "f"(tot_1) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4315,9 +4554,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_16 = offset_1 + sfx_1[0]; if (t_40 < clen) { { - auto _gst_310 = dg_base + goff_16; - __builtin_assume(__isGlobal(_gst_310)); - *(reinterpret_cast(_gst_310) + (0)) = dg_16; + auto _gst_342 = dg_base + goff_16; + __builtin_assume(__isGlobal(_gst_342)); + *(reinterpret_cast(_gst_342) + (0)) = dg_16; } } int t_41 = 16 + 8 * th + 1; @@ -4325,9 +4564,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_17 = offset_1 + sfx_1[1]; if (t_41 < clen) { { - auto _gst_311 = dg_base + goff_17; - __builtin_assume(__isGlobal(_gst_311)); - *(reinterpret_cast(_gst_311) + (0)) = dg_17; + auto _gst_343 = dg_base + goff_17; + __builtin_assume(__isGlobal(_gst_343)); + *(reinterpret_cast(_gst_343) + (0)) = dg_17; } } int t_42 = 16 + 8 * th + 2; @@ -4335,9 +4574,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_18 = offset_1 + sfx_1[2]; if (t_42 < clen) { { - auto _gst_312 = dg_base + goff_18; - __builtin_assume(__isGlobal(_gst_312)); - *(reinterpret_cast(_gst_312) + (0)) = dg_18; + auto _gst_344 = dg_base + goff_18; + __builtin_assume(__isGlobal(_gst_344)); + *(reinterpret_cast(_gst_344) + (0)) = dg_18; } } int t_43 = 16 + 8 * th + 3; @@ -4345,9 +4584,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_19 = offset_1 + sfx_1[3]; if (t_43 < clen) { { - auto _gst_313 = dg_base + goff_19; - __builtin_assume(__isGlobal(_gst_313)); - *(reinterpret_cast(_gst_313) + (0)) = dg_19; + auto _gst_345 = dg_base + goff_19; + __builtin_assume(__isGlobal(_gst_345)); + *(reinterpret_cast(_gst_345) + (0)) = dg_19; } } int t_44 = 16 + 8 * th + 4; @@ -4355,9 +4594,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_20 = offset_1 + sfx_1[4]; if (t_44 < clen) { { - auto _gst_314 = dg_base + goff_20; - __builtin_assume(__isGlobal(_gst_314)); - *(reinterpret_cast(_gst_314) + (0)) = dg_20; + auto _gst_346 = dg_base + goff_20; + __builtin_assume(__isGlobal(_gst_346)); + *(reinterpret_cast(_gst_346) + (0)) = dg_20; } } int t_45 = 16 + 8 * th + 5; @@ -4365,9 +4604,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_21 = offset_1 + sfx_1[5]; if (t_45 < clen) { { - auto _gst_315 = dg_base + goff_21; - __builtin_assume(__isGlobal(_gst_315)); - *(reinterpret_cast(_gst_315) + (0)) = dg_21; + auto _gst_347 = dg_base + goff_21; + __builtin_assume(__isGlobal(_gst_347)); + *(reinterpret_cast(_gst_347) + (0)) = dg_21; } } int t_46 = 16 + 8 * th + 6; @@ -4375,9 +4614,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_22 = offset_1 + sfx_1[6]; if (t_46 < clen) { { - auto _gst_316 = dg_base + goff_22; - __builtin_assume(__isGlobal(_gst_316)); - *(reinterpret_cast(_gst_316) + (0)) = dg_22; + auto _gst_348 = dg_base + goff_22; + __builtin_assume(__isGlobal(_gst_348)); + *(reinterpret_cast(_gst_348) + (0)) = dg_22; } } int t_47 = 16 + 8 * th + 7; @@ -4385,15 +4624,15 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_23 = offset_1 + sfx_1[7]; if (t_47 < clen) { { - auto _gst_317 = dg_base + goff_23; - __builtin_assume(__isGlobal(_gst_317)); - *(reinterpret_cast(_gst_317) + (0)) = dg_23; + auto _gst_349 = dg_base + goff_23; + __builtin_assume(__isGlobal(_gst_349)); + *(reinterpret_cast(_gst_349) + (0)) = dg_23; } } off_acc[0] = off_acc[0] + t_hi_1 + t_lo_1; { - uint32_t _addr_318 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_318), "f"(off_acc[0]) : "memory"); + uint32_t _addr_350 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_350), "f"(off_acc[0]) : "memory"); } mbarrier_wait(epi_full_addr + (th) * 8, 0); asm volatile("tcgen05.fence::after_thread_sync;"); @@ -4434,9 +4673,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_325[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_24 = __uint_as_float(r_325[0] << 16); float q_t0_24 = ((t_48 >= clen) ? 0.0f : q_t_24); - float dqf_24 = pool_f32[ew_2 + d_lane]; - float dkf_24 = pool_f32[ew_2 + 1024 + d_lane]; - float dgf_24 = pool_f32[ew_2 + 2048 + d_lane]; float bt_t_24 = btv_2[0]; float e_off_24 = approx_exp2(g_t_24 - gn_2); float e_dg_24 = approx_exp2(g_t_24 - gc_6); @@ -4444,22 +4680,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_24 = kdg_2[0] * e_dg_24; dbp_2[0] = dk2_24 * k_t_24; float dk2b_24 = dk2_24 * bt_t_24; + float q_r_24 = q_t0_24; + float qe_23 = q_r_24 * e_dg_24; + float kp_39 = k_t_24 * bt_t_24; + __nv_bfloat16 kb_23 = (__nv_bfloat16)kp_39; + float kbe_23 = (float)kb_23 * e_dg_24; + float cr_23 = __fmaf_rn(__uint_as_float(__as_u32(qe_23) & 4294959104u) - qe_23, qdg_2[0], (__uint_as_float(__as_u32(kbe_23) & 4294959104u) - kp_39 * e_dg_24) * kdg_2[0]); + float dqf_24 = pool_f32[ew_2 + d_lane]; + float dkf_24 = pool_f32[ew_2 + 1024 + d_lane]; + float dgf_24 = pool_f32[ew_2 + 2048 + d_lane]; float dq_o_24 = dq2_24 + dqf_24; float dk_o_24 = dk2b_24 + dkf_24 + dkt_6[0]; - float q_r_24 = q_t0_24; - float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, (dk2b_24 - dkt_6[0]) * k_t_24)) + dgf_24; + float dg_o_24 = __fmaf_rn(q_r_24, dqf_24, __fmaf_rn(q_r_24, dq2_24, __fmaf_rn(dk2b_24 - dkt_6[0], k_t_24, cr_23))) + dgf_24; dgo_2[0] = dg_o_24; int qoff_24 = t_48 * qrs; if (t_48 < clen) { { - auto _gst_319 = dq_base + qoff_24; - __builtin_assume(__isGlobal(_gst_319)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_319) + (0)) = (__nv_bfloat16)dq_o_24; + auto _gst_351 = dq_base + qoff_24; + __builtin_assume(__isGlobal(_gst_351)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_351) + (0)) = (__nv_bfloat16)dq_o_24; } { - auto _gst_320 = dk_base + qoff_24; - __builtin_assume(__isGlobal(_gst_320)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_320) + (0)) = (__nv_bfloat16)dk_o_24; + auto _gst_352 = dk_base + qoff_24; + __builtin_assume(__isGlobal(_gst_352)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_352) + (0)) = (__nv_bfloat16)dk_o_24; } } int t_49 = 8 * th + 1; @@ -4473,9 +4717,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_328[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 128 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 128 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_25 = __uint_as_float(r_328[0] << 16); float q_t0_25 = ((t_49 >= clen) ? 0.0f : q_t_25); - float dqf_25 = pool_f32[ew_2 + 128 + d_lane]; - float dkf_25 = pool_f32[ew_2 + 1152 + d_lane]; - float dgf_25 = pool_f32[ew_2 + 2176 + d_lane]; float bt_t_25 = btv_2[1]; float e_off_25 = approx_exp2(g_t_25 - gn_2); float e_dg_25 = approx_exp2(g_t_25 - gc_6); @@ -4483,22 +4724,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_25 = kdg_2[1] * e_dg_25; dbp_2[1] = dk2_25 * k_t_25; float dk2b_25 = dk2_25 * bt_t_25; + float q_r_25 = q_t0_25; + float qe_24 = q_r_25 * e_dg_25; + float kp_40 = k_t_25 * bt_t_25; + __nv_bfloat16 kb_24 = (__nv_bfloat16)kp_40; + float kbe_24 = (float)kb_24 * e_dg_25; + float cr_24 = __fmaf_rn(__uint_as_float(__as_u32(qe_24) & 4294959104u) - qe_24, qdg_2[1], (__uint_as_float(__as_u32(kbe_24) & 4294959104u) - kp_40 * e_dg_25) * kdg_2[1]); + float dqf_25 = pool_f32[ew_2 + 128 + d_lane]; + float dkf_25 = pool_f32[ew_2 + 1152 + d_lane]; + float dgf_25 = pool_f32[ew_2 + 2176 + d_lane]; float dq_o_25 = dq2_25 + dqf_25; float dk_o_25 = dk2b_25 + dkf_25 + dkt_6[1]; - float q_r_25 = q_t0_25; - float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, (dk2b_25 - dkt_6[1]) * k_t_25)) + dgf_25; + float dg_o_25 = __fmaf_rn(q_r_25, dqf_25, __fmaf_rn(q_r_25, dq2_25, __fmaf_rn(dk2b_25 - dkt_6[1], k_t_25, cr_24))) + dgf_25; dgo_2[1] = dg_o_25; int qoff_25 = t_49 * qrs; if (t_49 < clen) { { - auto _gst_321 = dq_base + qoff_25; - __builtin_assume(__isGlobal(_gst_321)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_321) + (0)) = (__nv_bfloat16)dq_o_25; + auto _gst_353 = dq_base + qoff_25; + __builtin_assume(__isGlobal(_gst_353)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_353) + (0)) = (__nv_bfloat16)dq_o_25; } { - auto _gst_322 = dk_base + qoff_25; - __builtin_assume(__isGlobal(_gst_322)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_322) + (0)) = (__nv_bfloat16)dk_o_25; + auto _gst_354 = dk_base + qoff_25; + __builtin_assume(__isGlobal(_gst_354)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_354) + (0)) = (__nv_bfloat16)dk_o_25; } } int t_50 = 8 * th + 2; @@ -4512,9 +4761,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_331[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 256 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 256 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_26 = __uint_as_float(r_331[0] << 16); float q_t0_26 = ((t_50 >= clen) ? 0.0f : q_t_26); - float dqf_26 = pool_f32[ew_2 + 256 + d_lane]; - float dkf_26 = pool_f32[ew_2 + 1280 + d_lane]; - float dgf_26 = pool_f32[ew_2 + 2304 + d_lane]; float bt_t_26 = btv_2[2]; float e_off_26 = approx_exp2(g_t_26 - gn_2); float e_dg_26 = approx_exp2(g_t_26 - gc_6); @@ -4522,22 +4768,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_26 = kdg_2[2] * e_dg_26; dbp_2[2] = dk2_26 * k_t_26; float dk2b_26 = dk2_26 * bt_t_26; + float q_r_26 = q_t0_26; + float qe_25 = q_r_26 * e_dg_26; + float kp_41 = k_t_26 * bt_t_26; + __nv_bfloat16 kb_25 = (__nv_bfloat16)kp_41; + float kbe_25 = (float)kb_25 * e_dg_26; + float cr_25 = __fmaf_rn(__uint_as_float(__as_u32(qe_25) & 4294959104u) - qe_25, qdg_2[2], (__uint_as_float(__as_u32(kbe_25) & 4294959104u) - kp_41 * e_dg_26) * kdg_2[2]); + float dqf_26 = pool_f32[ew_2 + 256 + d_lane]; + float dkf_26 = pool_f32[ew_2 + 1280 + d_lane]; + float dgf_26 = pool_f32[ew_2 + 2304 + d_lane]; float dq_o_26 = dq2_26 + dqf_26; float dk_o_26 = dk2b_26 + dkf_26 + dkt_6[2]; - float q_r_26 = q_t0_26; - float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, (dk2b_26 - dkt_6[2]) * k_t_26)) + dgf_26; + float dg_o_26 = __fmaf_rn(q_r_26, dqf_26, __fmaf_rn(q_r_26, dq2_26, __fmaf_rn(dk2b_26 - dkt_6[2], k_t_26, cr_25))) + dgf_26; dgo_2[2] = dg_o_26; int qoff_26 = t_50 * qrs; if (t_50 < clen) { { - auto _gst_323 = dq_base + qoff_26; - __builtin_assume(__isGlobal(_gst_323)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_323) + (0)) = (__nv_bfloat16)dq_o_26; + auto _gst_355 = dq_base + qoff_26; + __builtin_assume(__isGlobal(_gst_355)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_355) + (0)) = (__nv_bfloat16)dq_o_26; } { - auto _gst_324 = dk_base + qoff_26; - __builtin_assume(__isGlobal(_gst_324)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_324) + (0)) = (__nv_bfloat16)dk_o_26; + auto _gst_356 = dk_base + qoff_26; + __builtin_assume(__isGlobal(_gst_356)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_356) + (0)) = (__nv_bfloat16)dk_o_26; } } int t_51 = 8 * th + 3; @@ -4551,9 +4805,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_334[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 384 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 384 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_27 = __uint_as_float(r_334[0] << 16); float q_t0_27 = ((t_51 >= clen) ? 0.0f : q_t_27); - float dqf_27 = pool_f32[ew_2 + 384 + d_lane]; - float dkf_27 = pool_f32[ew_2 + 1408 + d_lane]; - float dgf_27 = pool_f32[ew_2 + 2432 + d_lane]; float bt_t_27 = btv_2[3]; float e_off_27 = approx_exp2(g_t_27 - gn_2); float e_dg_27 = approx_exp2(g_t_27 - gc_6); @@ -4561,22 +4812,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_27 = kdg_2[3] * e_dg_27; dbp_2[3] = dk2_27 * k_t_27; float dk2b_27 = dk2_27 * bt_t_27; + float q_r_27 = q_t0_27; + float qe_26 = q_r_27 * e_dg_27; + float kp_42 = k_t_27 * bt_t_27; + __nv_bfloat16 kb_26 = (__nv_bfloat16)kp_42; + float kbe_26 = (float)kb_26 * e_dg_27; + float cr_26 = __fmaf_rn(__uint_as_float(__as_u32(qe_26) & 4294959104u) - qe_26, qdg_2[3], (__uint_as_float(__as_u32(kbe_26) & 4294959104u) - kp_42 * e_dg_27) * kdg_2[3]); + float dqf_27 = pool_f32[ew_2 + 384 + d_lane]; + float dkf_27 = pool_f32[ew_2 + 1408 + d_lane]; + float dgf_27 = pool_f32[ew_2 + 2432 + d_lane]; float dq_o_27 = dq2_27 + dqf_27; float dk_o_27 = dk2b_27 + dkf_27 + dkt_6[3]; - float q_r_27 = q_t0_27; - float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, (dk2b_27 - dkt_6[3]) * k_t_27)) + dgf_27; + float dg_o_27 = __fmaf_rn(q_r_27, dqf_27, __fmaf_rn(q_r_27, dq2_27, __fmaf_rn(dk2b_27 - dkt_6[3], k_t_27, cr_26))) + dgf_27; dgo_2[3] = dg_o_27; int qoff_27 = t_51 * qrs; if (t_51 < clen) { { - auto _gst_325 = dq_base + qoff_27; - __builtin_assume(__isGlobal(_gst_325)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_325) + (0)) = (__nv_bfloat16)dq_o_27; + auto _gst_357 = dq_base + qoff_27; + __builtin_assume(__isGlobal(_gst_357)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_357) + (0)) = (__nv_bfloat16)dq_o_27; } { - auto _gst_326 = dk_base + qoff_27; - __builtin_assume(__isGlobal(_gst_326)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_326) + (0)) = (__nv_bfloat16)dk_o_27; + auto _gst_358 = dk_base + qoff_27; + __builtin_assume(__isGlobal(_gst_358)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_358) + (0)) = (__nv_bfloat16)dk_o_27; } } int t_52 = 8 * th + 4; @@ -4590,9 +4849,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_337[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 512 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 512 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_28 = __uint_as_float(r_337[0] << 16); float q_t0_28 = ((t_52 >= clen) ? 0.0f : q_t_28); - float dqf_28 = pool_f32[ew_2 + 512 + d_lane]; - float dkf_28 = pool_f32[ew_2 + 1536 + d_lane]; - float dgf_28 = pool_f32[ew_2 + 2560 + d_lane]; float bt_t_28 = btv_2[4]; float e_off_28 = approx_exp2(g_t_28 - gn_2); float e_dg_28 = approx_exp2(g_t_28 - gc_6); @@ -4600,22 +4856,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_28 = kdg_2[4] * e_dg_28; dbp_2[4] = dk2_28 * k_t_28; float dk2b_28 = dk2_28 * bt_t_28; + float q_r_28 = q_t0_28; + float qe_27 = q_r_28 * e_dg_28; + float kp_43 = k_t_28 * bt_t_28; + __nv_bfloat16 kb_27 = (__nv_bfloat16)kp_43; + float kbe_27 = (float)kb_27 * e_dg_28; + float cr_27 = __fmaf_rn(__uint_as_float(__as_u32(qe_27) & 4294959104u) - qe_27, qdg_2[4], (__uint_as_float(__as_u32(kbe_27) & 4294959104u) - kp_43 * e_dg_28) * kdg_2[4]); + float dqf_28 = pool_f32[ew_2 + 512 + d_lane]; + float dkf_28 = pool_f32[ew_2 + 1536 + d_lane]; + float dgf_28 = pool_f32[ew_2 + 2560 + d_lane]; float dq_o_28 = dq2_28 + dqf_28; float dk_o_28 = dk2b_28 + dkf_28 + dkt_6[4]; - float q_r_28 = q_t0_28; - float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, (dk2b_28 - dkt_6[4]) * k_t_28)) + dgf_28; + float dg_o_28 = __fmaf_rn(q_r_28, dqf_28, __fmaf_rn(q_r_28, dq2_28, __fmaf_rn(dk2b_28 - dkt_6[4], k_t_28, cr_27))) + dgf_28; dgo_2[4] = dg_o_28; int qoff_28 = t_52 * qrs; if (t_52 < clen) { { - auto _gst_327 = dq_base + qoff_28; - __builtin_assume(__isGlobal(_gst_327)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_327) + (0)) = (__nv_bfloat16)dq_o_28; + auto _gst_359 = dq_base + qoff_28; + __builtin_assume(__isGlobal(_gst_359)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_359) + (0)) = (__nv_bfloat16)dq_o_28; } { - auto _gst_328 = dk_base + qoff_28; - __builtin_assume(__isGlobal(_gst_328)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_328) + (0)) = (__nv_bfloat16)dk_o_28; + auto _gst_360 = dk_base + qoff_28; + __builtin_assume(__isGlobal(_gst_360)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_360) + (0)) = (__nv_bfloat16)dk_o_28; } } int t_53 = 8 * th + 5; @@ -4629,9 +4893,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_340[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 640 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 640 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_29 = __uint_as_float(r_340[0] << 16); float q_t0_29 = ((t_53 >= clen) ? 0.0f : q_t_29); - float dqf_29 = pool_f32[ew_2 + 640 + d_lane]; - float dkf_29 = pool_f32[ew_2 + 1664 + d_lane]; - float dgf_29 = pool_f32[ew_2 + 2688 + d_lane]; float bt_t_29 = btv_2[5]; float e_off_29 = approx_exp2(g_t_29 - gn_2); float e_dg_29 = approx_exp2(g_t_29 - gc_6); @@ -4639,22 +4900,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_29 = kdg_2[5] * e_dg_29; dbp_2[5] = dk2_29 * k_t_29; float dk2b_29 = dk2_29 * bt_t_29; + float q_r_29 = q_t0_29; + float qe_28 = q_r_29 * e_dg_29; + float kp_44 = k_t_29 * bt_t_29; + __nv_bfloat16 kb_28 = (__nv_bfloat16)kp_44; + float kbe_28 = (float)kb_28 * e_dg_29; + float cr_28 = __fmaf_rn(__uint_as_float(__as_u32(qe_28) & 4294959104u) - qe_28, qdg_2[5], (__uint_as_float(__as_u32(kbe_28) & 4294959104u) - kp_44 * e_dg_29) * kdg_2[5]); + float dqf_29 = pool_f32[ew_2 + 640 + d_lane]; + float dkf_29 = pool_f32[ew_2 + 1664 + d_lane]; + float dgf_29 = pool_f32[ew_2 + 2688 + d_lane]; float dq_o_29 = dq2_29 + dqf_29; float dk_o_29 = dk2b_29 + dkf_29 + dkt_6[5]; - float q_r_29 = q_t0_29; - float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, (dk2b_29 - dkt_6[5]) * k_t_29)) + dgf_29; + float dg_o_29 = __fmaf_rn(q_r_29, dqf_29, __fmaf_rn(q_r_29, dq2_29, __fmaf_rn(dk2b_29 - dkt_6[5], k_t_29, cr_28))) + dgf_29; dgo_2[5] = dg_o_29; int qoff_29 = t_53 * qrs; if (t_53 < clen) { { - auto _gst_329 = dq_base + qoff_29; - __builtin_assume(__isGlobal(_gst_329)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_329) + (0)) = (__nv_bfloat16)dq_o_29; + auto _gst_361 = dq_base + qoff_29; + __builtin_assume(__isGlobal(_gst_361)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_361) + (0)) = (__nv_bfloat16)dq_o_29; } { - auto _gst_330 = dk_base + qoff_29; - __builtin_assume(__isGlobal(_gst_330)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_330) + (0)) = (__nv_bfloat16)dk_o_29; + auto _gst_362 = dk_base + qoff_29; + __builtin_assume(__isGlobal(_gst_362)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_362) + (0)) = (__nv_bfloat16)dk_o_29; } } int t_54 = 8 * th + 6; @@ -4668,9 +4937,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_343[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 768 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 768 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_30 = __uint_as_float(r_343[0] << 16); float q_t0_30 = ((t_54 >= clen) ? 0.0f : q_t_30); - float dqf_30 = pool_f32[ew_2 + 768 + d_lane]; - float dkf_30 = pool_f32[ew_2 + 1792 + d_lane]; - float dgf_30 = pool_f32[ew_2 + 2816 + d_lane]; float bt_t_30 = btv_2[6]; float e_off_30 = approx_exp2(g_t_30 - gn_2); float e_dg_30 = approx_exp2(g_t_30 - gc_6); @@ -4678,22 +4944,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_30 = kdg_2[6] * e_dg_30; dbp_2[6] = dk2_30 * k_t_30; float dk2b_30 = dk2_30 * bt_t_30; + float q_r_30 = q_t0_30; + float qe_29 = q_r_30 * e_dg_30; + float kp_45 = k_t_30 * bt_t_30; + __nv_bfloat16 kb_29 = (__nv_bfloat16)kp_45; + float kbe_29 = (float)kb_29 * e_dg_30; + float cr_29 = __fmaf_rn(__uint_as_float(__as_u32(qe_29) & 4294959104u) - qe_29, qdg_2[6], (__uint_as_float(__as_u32(kbe_29) & 4294959104u) - kp_45 * e_dg_30) * kdg_2[6]); + float dqf_30 = pool_f32[ew_2 + 768 + d_lane]; + float dkf_30 = pool_f32[ew_2 + 1792 + d_lane]; + float dgf_30 = pool_f32[ew_2 + 2816 + d_lane]; float dq_o_30 = dq2_30 + dqf_30; float dk_o_30 = dk2b_30 + dkf_30 + dkt_6[6]; - float q_r_30 = q_t0_30; - float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, (dk2b_30 - dkt_6[6]) * k_t_30)) + dgf_30; + float dg_o_30 = __fmaf_rn(q_r_30, dqf_30, __fmaf_rn(q_r_30, dq2_30, __fmaf_rn(dk2b_30 - dkt_6[6], k_t_30, cr_29))) + dgf_30; dgo_2[6] = dg_o_30; int qoff_30 = t_54 * qrs; if (t_54 < clen) { { - auto _gst_331 = dq_base + qoff_30; - __builtin_assume(__isGlobal(_gst_331)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_331) + (0)) = (__nv_bfloat16)dq_o_30; + auto _gst_363 = dq_base + qoff_30; + __builtin_assume(__isGlobal(_gst_363)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_363) + (0)) = (__nv_bfloat16)dq_o_30; } { - auto _gst_332 = dk_base + qoff_30; - __builtin_assume(__isGlobal(_gst_332)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_332) + (0)) = (__nv_bfloat16)dk_o_30; + auto _gst_364 = dk_base + qoff_30; + __builtin_assume(__isGlobal(_gst_364)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_364) + (0)) = (__nv_bfloat16)dk_o_30; } } int t_55 = 8 * th + 7; @@ -4707,9 +4981,6 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d asm volatile("ld.shared.u16 %0, [%1];" : "=r"(*reinterpret_cast(&r_346[0])) : "r"((qbase_6 + (dcol / 64 * 1024 + 896 + dcol % 64 * 2 ^ (dcol / 64 * 1024 + 896 + dcol % 64 * 2 >> 7 & 7) << 4)))); float q_t_31 = __uint_as_float(r_346[0] << 16); float q_t0_31 = ((t_55 >= clen) ? 0.0f : q_t_31); - float dqf_31 = pool_f32[ew_2 + 896 + d_lane]; - float dkf_31 = pool_f32[ew_2 + 1920 + d_lane]; - float dgf_31 = pool_f32[ew_2 + 2944 + d_lane]; float bt_t_31 = btv_2[7]; float e_off_31 = approx_exp2(g_t_31 - gn_2); float e_dg_31 = approx_exp2(g_t_31 - gc_6); @@ -4717,22 +4988,30 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dk2_31 = kdg_2[7] * e_dg_31; dbp_2[7] = dk2_31 * k_t_31; float dk2b_31 = dk2_31 * bt_t_31; + float q_r_31 = q_t0_31; + float qe_30 = q_r_31 * e_dg_31; + float kp_46 = k_t_31 * bt_t_31; + __nv_bfloat16 kb_30 = (__nv_bfloat16)kp_46; + float kbe_30 = (float)kb_30 * e_dg_31; + float cr_30 = __fmaf_rn(__uint_as_float(__as_u32(qe_30) & 4294959104u) - qe_30, qdg_2[7], (__uint_as_float(__as_u32(kbe_30) & 4294959104u) - kp_46 * e_dg_31) * kdg_2[7]); + float dqf_31 = pool_f32[ew_2 + 896 + d_lane]; + float dkf_31 = pool_f32[ew_2 + 1920 + d_lane]; + float dgf_31 = pool_f32[ew_2 + 2944 + d_lane]; float dq_o_31 = dq2_31 + dqf_31; float dk_o_31 = dk2b_31 + dkf_31 + dkt_6[7]; - float q_r_31 = q_t0_31; - float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, (dk2b_31 - dkt_6[7]) * k_t_31)) + dgf_31; + float dg_o_31 = __fmaf_rn(q_r_31, dqf_31, __fmaf_rn(q_r_31, dq2_31, __fmaf_rn(dk2b_31 - dkt_6[7], k_t_31, cr_30))) + dgf_31; dgo_2[7] = dg_o_31; int qoff_31 = t_55 * qrs; if (t_55 < clen) { { - auto _gst_333 = dq_base + qoff_31; - __builtin_assume(__isGlobal(_gst_333)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_333) + (0)) = (__nv_bfloat16)dq_o_31; + auto _gst_365 = dq_base + qoff_31; + __builtin_assume(__isGlobal(_gst_365)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_365) + (0)) = (__nv_bfloat16)dq_o_31; } { - auto _gst_334 = dk_base + qoff_31; - __builtin_assume(__isGlobal(_gst_334)); - *(reinterpret_cast<__nv_bfloat16*>(_gst_334) + (0)) = (__nv_bfloat16)dk_o_31; + auto _gst_366 = dk_base + qoff_31; + __builtin_assume(__isGlobal(_gst_366)); + *(reinterpret_cast<__nv_bfloat16*>(_gst_366) + (0)) = (__nv_bfloat16)dk_o_31; } } float s_run_2[1]; @@ -4761,35 +5040,35 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d int b2_2 = lane_1 >> 2 & 1; float v4_2[4]; float send_20 = ((b4_2 == 0) ? dbp_2[4] : dbp_2[0]); - float _shfl_xor_sync_335 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); - v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_335; + float _shfl_xor_sync_367 = __shfl_xor_sync(0xFFFFFFFF, send_20, 16); + v4_2[0] = ((b4_2 == 0) ? dbp_2[0] : dbp_2[4]) + _shfl_xor_sync_367; float send_21 = ((b4_2 == 0) ? dbp_2[5] : dbp_2[1]); - float _shfl_xor_sync_336 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); - v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_336; + float _shfl_xor_sync_368 = __shfl_xor_sync(0xFFFFFFFF, send_21, 16); + v4_2[1] = ((b4_2 == 0) ? dbp_2[1] : dbp_2[5]) + _shfl_xor_sync_368; float send_22 = ((b4_2 == 0) ? dbp_2[6] : dbp_2[2]); - float _shfl_xor_sync_337 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); - v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_337; + float _shfl_xor_sync_369 = __shfl_xor_sync(0xFFFFFFFF, send_22, 16); + v4_2[2] = ((b4_2 == 0) ? dbp_2[2] : dbp_2[6]) + _shfl_xor_sync_369; float send_23 = ((b4_2 == 0) ? dbp_2[7] : dbp_2[3]); - float _shfl_xor_sync_338 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); - v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_338; + float _shfl_xor_sync_370 = __shfl_xor_sync(0xFFFFFFFF, send_23, 16); + v4_2[3] = ((b4_2 == 0) ? dbp_2[3] : dbp_2[7]) + _shfl_xor_sync_370; float v2_26[2]; float send_24 = ((b3_2 == 0) ? v4_2[2] : v4_2[0]); - float _shfl_xor_sync_339 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); - v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_339; + float _shfl_xor_sync_371 = __shfl_xor_sync(0xFFFFFFFF, send_24, 8); + v2_26[0] = ((b3_2 == 0) ? v4_2[0] : v4_2[2]) + _shfl_xor_sync_371; float send_25 = ((b3_2 == 0) ? v4_2[3] : v4_2[1]); - float _shfl_xor_sync_340 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); - v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_340; + float _shfl_xor_sync_372 = __shfl_xor_sync(0xFFFFFFFF, send_25, 8); + v2_26[1] = ((b3_2 == 0) ? v4_2[1] : v4_2[3]) + _shfl_xor_sync_372; float send_26 = ((b2_2 == 0) ? v2_26[1] : v2_26[0]); - float _shfl_xor_sync_341 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); - float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_341; - float _shfl_xor_sync_342 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); - v1_2 = v1_2 + _shfl_xor_sync_342; - float _shfl_xor_sync_343 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); - v1_2 = v1_2 + _shfl_xor_sync_343; + float _shfl_xor_sync_373 = __shfl_xor_sync(0xFFFFFFFF, send_26, 4); + float v1_2 = ((b2_2 == 0) ? v2_26[0] : v2_26[1]) + _shfl_xor_sync_373; + float _shfl_xor_sync_374 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 2); + v1_2 = v1_2 + _shfl_xor_sync_374; + float _shfl_xor_sync_375 = __shfl_xor_sync(0xFFFFFFFF, v1_2, 1); + v1_2 = v1_2 + _shfl_xor_sync_375; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&off_acc[0])) : "r"(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1)))); { - uint32_t _addr_344 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); - asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_344), "f"(tot_2) : "memory"); + uint32_t _addr_376 = static_cast(pool_f32_addr + (unsigned int)(113664 + 4 * (qd * 64 + th * 32 + lane_1))); + asm volatile("st.shared.f32 [%0], %1;" :: "r"(_addr_376), "f"(tot_2) : "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); asm volatile("barrier.sync %0, 64;" :: "r"(2 + qd) : "memory"); @@ -4807,9 +5086,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_24 = offset_2 + sfx_2[0]; if (t_56 < clen) { { - auto _gst_345 = dg_base + goff_24; - __builtin_assume(__isGlobal(_gst_345)); - *(reinterpret_cast(_gst_345) + (0)) = dg_24; + auto _gst_377 = dg_base + goff_24; + __builtin_assume(__isGlobal(_gst_377)); + *(reinterpret_cast(_gst_377) + (0)) = dg_24; } } int t_57 = 8 * th + 1; @@ -4817,9 +5096,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_25 = offset_2 + sfx_2[1]; if (t_57 < clen) { { - auto _gst_346 = dg_base + goff_25; - __builtin_assume(__isGlobal(_gst_346)); - *(reinterpret_cast(_gst_346) + (0)) = dg_25; + auto _gst_378 = dg_base + goff_25; + __builtin_assume(__isGlobal(_gst_378)); + *(reinterpret_cast(_gst_378) + (0)) = dg_25; } } int t_58 = 8 * th + 2; @@ -4827,9 +5106,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_26 = offset_2 + sfx_2[2]; if (t_58 < clen) { { - auto _gst_347 = dg_base + goff_26; - __builtin_assume(__isGlobal(_gst_347)); - *(reinterpret_cast(_gst_347) + (0)) = dg_26; + auto _gst_379 = dg_base + goff_26; + __builtin_assume(__isGlobal(_gst_379)); + *(reinterpret_cast(_gst_379) + (0)) = dg_26; } } int t_59 = 8 * th + 3; @@ -4837,9 +5116,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_27 = offset_2 + sfx_2[3]; if (t_59 < clen) { { - auto _gst_348 = dg_base + goff_27; - __builtin_assume(__isGlobal(_gst_348)); - *(reinterpret_cast(_gst_348) + (0)) = dg_27; + auto _gst_380 = dg_base + goff_27; + __builtin_assume(__isGlobal(_gst_380)); + *(reinterpret_cast(_gst_380) + (0)) = dg_27; } } int t_60 = 8 * th + 4; @@ -4847,9 +5126,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_28 = offset_2 + sfx_2[4]; if (t_60 < clen) { { - auto _gst_349 = dg_base + goff_28; - __builtin_assume(__isGlobal(_gst_349)); - *(reinterpret_cast(_gst_349) + (0)) = dg_28; + auto _gst_381 = dg_base + goff_28; + __builtin_assume(__isGlobal(_gst_381)); + *(reinterpret_cast(_gst_381) + (0)) = dg_28; } } int t_61 = 8 * th + 5; @@ -4857,9 +5136,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_29 = offset_2 + sfx_2[5]; if (t_61 < clen) { { - auto _gst_350 = dg_base + goff_29; - __builtin_assume(__isGlobal(_gst_350)); - *(reinterpret_cast(_gst_350) + (0)) = dg_29; + auto _gst_382 = dg_base + goff_29; + __builtin_assume(__isGlobal(_gst_382)); + *(reinterpret_cast(_gst_382) + (0)) = dg_29; } } int t_62 = 8 * th + 6; @@ -4867,9 +5146,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_30 = offset_2 + sfx_2[6]; if (t_62 < clen) { { - auto _gst_351 = dg_base + goff_30; - __builtin_assume(__isGlobal(_gst_351)); - *(reinterpret_cast(_gst_351) + (0)) = dg_30; + auto _gst_383 = dg_base + goff_30; + __builtin_assume(__isGlobal(_gst_383)); + *(reinterpret_cast(_gst_383) + (0)) = dg_30; } } int t_63 = 8 * th + 7; @@ -4877,9 +5156,9 @@ kernel_cake_kda_train_f4ea302627f05d805437(const __grid_constant__ CUtensorMap d float dg_31 = offset_2 + sfx_2[7]; if (t_63 < clen) { { - auto _gst_352 = dg_base + goff_31; - __builtin_assume(__isGlobal(_gst_352)); - *(reinterpret_cast(_gst_352) + (0)) = dg_31; + auto _gst_384 = dg_base + goff_31; + __builtin_assume(__isGlobal(_gst_384)); + *(reinterpret_cast(_gst_384) + (0)) = dg_31; } } off_acc[0] = off_acc[0] + t_hi_2 + t_lo_2; From b7f34ac1cae350c61407f8cdc1f78f998fc3dc06 Mon Sep 17 00:00:00 2001 From: Yingyi Huang Date: Thu, 5 Nov 2026 17:48:37 -0800 Subject: [PATCH 4/5] cake_kda_det_train: CUDA 12.9 floor for the sm_100a entries that use PTX ISA 8.8 instructions The sm_100a entries whose kernels use the 256-bit ld / st or the three-input min / max declared min_cuda 12.8, but nvcc 12.8 emits PTX ISA 8.7 and cannot build them. Those entries now declare min_cuda 12.9. Only build.json changes. --- .../cake_kda_det_train/build.json | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/flashinfer/experimental/cake_kda_det_train/build.json b/flashinfer/experimental/cake_kda_det_train/build.json index 4f32c1f7a93..6d353295534 100644 --- a/flashinfer/experimental/cake_kda_det_train/build.json +++ b/flashinfer/experimental/cake_kda_det_train/build.json @@ -128,7 +128,7 @@ "closure_sha256": "be78da9cf92f6390c7abce6079b23a01c20b0c417e88182f0a9dfa56899f12ea", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_036e1c7e1dfc95a88aae_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -191,7 +191,7 @@ "closure_sha256": "01f3800933989cb908e2d41cd308d5f1299e7c60b67b135fcd6b6872055cb482", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_058a68a5b4d89c53be4b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -842,7 +842,7 @@ "closure_sha256": "36e389cfe8bb3e87d3283b12f2ca6344d53605eb8f9ccd77e971630e669e98a9", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_1f9c46f98f4290ee26df_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -968,7 +968,7 @@ "closure_sha256": "e2adf85405786dba245894592541739d1dfd8b13d9b7b2559a3f3673a51a9221", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_22812d87c26b53a1ba10_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1031,7 +1031,7 @@ "closure_sha256": "290ff01688dfddc78f3b3aedf13ca2b8f3cb3a9d5a2031611d20888a16fb1916", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_22a2f04d41df18554b84_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1094,7 +1094,7 @@ "closure_sha256": "e169902bb5fffbab95950d7fd24116403986df3abb89fd1190a802c96013de7f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_243fb0b23aea1b296ab6_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1367,7 +1367,7 @@ "closure_sha256": "fb9a5b5ed6a3a643c231bff18c0274bae18ad4d1a309773bebc176decb9997bb", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_3712054f19d4180caa28_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1577,7 +1577,7 @@ "closure_sha256": "e47e8b8db302377a0fa3f1e57ffed611dad307b53fabc4babb91780de18be8d1", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_3f4b037a0a1f8dcc314b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1766,7 +1766,7 @@ "closure_sha256": "85e7e25b555d9017a2e3b18fbcb3d3c3a9d99dfdb183e1b34ba860ac52c24c62", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_4734a5d4f63d815ec086_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1913,7 +1913,7 @@ "closure_sha256": "ad53bdf6062f0005d086416f3425a74249b1d5e21ef02bda36f4c917eeed3ecf", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_4cce3cc5bc6210c95ce3_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2102,7 +2102,7 @@ "closure_sha256": "d5c36ecaff1392e62c22eda675af29ac282f1403e598f0b5090ea967fd4a3f01", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_5f5a5b3715675e083823_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2165,7 +2165,7 @@ "closure_sha256": "91178a228e0de03ef823845d47c1bf808a0d5d911203ce2963ef29688b26af5d", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_5fcc76904e5e2dd891de_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2354,7 +2354,7 @@ "closure_sha256": "d0677903f801235cc735df70a4571e4882e3cc034c6ff7de2ac25a6c8edbcf2a", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_706240be56d4e41df1a2_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2669,7 +2669,7 @@ "closure_sha256": "f1548497bb1c39094345d5e758e50544837610c16dd8a5aff7c385f83195952f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_7eca60b36e233943dae6_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2984,7 +2984,7 @@ "closure_sha256": "cf5e6f417b179ed37d478f879334ff5e82403160993eb2100d7a67310f9b1e5c", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_898a1eb972fbaf55789a_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3047,7 +3047,7 @@ "closure_sha256": "86bc258e380620ba45b64d1fe29d21b829fb1983741d571e3e8aebf3699109e5", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_8ab0929b8154c90b5eff_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3362,7 +3362,7 @@ "closure_sha256": "8c570885d73c86a9054988b8b792cb95c00a507716ed78a11513bc42e1b4a984", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_909a0aa57efb3f353972_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3488,7 +3488,7 @@ "closure_sha256": "7dbb4035f2c693851c8c7f15824acc32b42d3c65a5036ccecda66da5dd87b23f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_945a87fa84a265fc9f94_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3677,7 +3677,7 @@ "closure_sha256": "6ac7a7e3d9cf4572c7069a549540381c74954b1356fff208cd3713772e7e16e7", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_9931cdff5dfd86c687c7_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3824,7 +3824,7 @@ "closure_sha256": "f48e9fc3ec2e1e6ea7367e9c68bb1d65222329ab95a2f311ee5e21b85953a0b1", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_a557058a8aeca49a6402_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3971,7 +3971,7 @@ "closure_sha256": "0d270842881dfb70f68fa4a8e648163de22f0b7cfa96e436080d5d10f5017a36", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_ac974e171d56fc3ce10b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4622,7 +4622,7 @@ "closure_sha256": "08b1943d3b751202f87afd37d466ef82f4161f19dadbaeddfecec3ca744373d8", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_d32455627b87ba0a1d87_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4811,7 +4811,7 @@ "closure_sha256": "c64f17ffdd12bee3c4a4e1b930d146e6d7abde8a3802e40d039ef99887613ac9", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_dec885b973bd44f88416_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -5063,7 +5063,7 @@ "closure_sha256": "10c66cfb894c37ef5f81e7cda989d7fa3b50558497c449d6414861afa81c3201", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_e60e7ca5d88192193e61_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -5315,7 +5315,7 @@ "closure_sha256": "b30e9a22c26b09d6b4561307e4525f8f6a90944caa71989462e3624edd5ab86a", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_eac26fffb3df8780302c_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -5378,7 +5378,7 @@ "closure_sha256": "05ec3212fb39e9b1e8d81a29c8e6b5ff101bfc3c74a98f34ace71d031401ebb2", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_f011cd9add8f3c7c3eee_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -5504,7 +5504,7 @@ "closure_sha256": "925f0e20f2aa1d2b2e4eff092eaf4552651afa913c5d9a86d518deedd9d7ccc2", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_f6a70fb0288a508f2828_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -5630,7 +5630,7 @@ "closure_sha256": "f8a8577c36e3970ab36ec45b8f0686e2522826e35ac4b6227ce9fcf508e220e6", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_det_train_f92b256cccc2c9007554_sm_100a", "nvcc_flags": [ "--use_fast_math" From 99ed8cc8dce3605e5dd12bb525fe9db7c260d175 Mon Sep 17 00:00:00 2001 From: Yingyi Huang Date: Thu, 5 Nov 2026 18:50:44 -0800 Subject: [PATCH 5/5] cake_kda_train: CUDA 12.9 floor for the sm_100a entries that use PTX ISA 8.8 instructions The parent commit raises the deterministic package's sm_100a entries whose kernels use the 256-bit ld / st or the three-input min / max to min_cuda 12.9: nvcc 12.8 emits PTX ISA 8.7 and cannot build them. This package's copies of those programs (28 sm_100a entries) now declare the min_cuda of the deterministic entries they copy, and its own programs whose kernels use those instructions (3 sm_100a entries) take the same floor. Only build.json changes: 31 sm_100a entries 12.8 -> 12.9. The sm_103a and sm_107a floors (12.9, 13.4) are unchanged. A second render and a second copy pass change nothing. tests/experimental/test_cake_kda*.py on CPU: 2000 passed, 913 skipped (parent: 1 failed, test_copied_programs_are_the_deterministic_packages_sources, which holds the copies' min_cuda to the deterministic entries). --- .../cake/cake_kda_train/build.json | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/flashinfer/experimental/cake/cake_kda_train/build.json b/flashinfer/experimental/cake/cake_kda_train/build.json index 5e2215eb81d..985c627edc6 100644 --- a/flashinfer/experimental/cake/cake_kda_train/build.json +++ b/flashinfer/experimental/cake/cake_kda_train/build.json @@ -65,7 +65,7 @@ "closure_sha256": "fb6d85f446aab7615d68497e9cd913d3359fd69a7aa3e675f565e4971eb6bd0f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_036e1c7e1dfc95a88aae_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -128,7 +128,7 @@ "closure_sha256": "712d269fc7c10b333f398e75c4f90da33c474fc67e03bf6ad230e3d428303dc4", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_058a68a5b4d89c53be4b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -443,7 +443,7 @@ "closure_sha256": "6d77155481289eb946b62d56835ba8c6f3a38dd9c74a3cd309ac00faf3f53586", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_1563b1b6e66464bbfa33_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -779,7 +779,7 @@ "closure_sha256": "f3412fb76bc083c747fafef128c40b8e489d6d00d470316b48195301d9c005f7", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_1f9c46f98f4290ee26df_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -905,7 +905,7 @@ "closure_sha256": "bbf1df4e57974fd42c36cb8c50e9dc652c61757478ef61bdde58f352c61d1719", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_22812d87c26b53a1ba10_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -968,7 +968,7 @@ "closure_sha256": "1b9bcd117cb8abc20ecb45593f227b2feb0663130135b74d9e9e72efa7d53f6b", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_22a2f04d41df18554b84_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1031,7 +1031,7 @@ "closure_sha256": "24cbff914e1bd50ee1d302a8dee09f21a3f63d8df86c0a5ad244ca7675af2dca", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_243fb0b23aea1b296ab6_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1304,7 +1304,7 @@ "closure_sha256": "c918af9219b8b751a61a7c1ee66beba3585438ef393104384c42f4ebe1e423b4", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_3712054f19d4180caa28_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1451,7 +1451,7 @@ "closure_sha256": "3d3e8c8024ab208ab49493993bec65024882d3bfb7eac1fd153cb278721116a6", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_3f4b037a0a1f8dcc314b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1577,7 +1577,7 @@ "closure_sha256": "9e6dd40bd43b36d8c4dbfd5623ab6f8691cb92d1bc4ae9e4cc38e4d195c6c3bf", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_4734a5d4f63d815ec086_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1724,7 +1724,7 @@ "closure_sha256": "616262ba6230d246892ec9202f4dc573254c27226aca5c2d5cee736f5384891d", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_4cce3cc5bc6210c95ce3_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1871,7 +1871,7 @@ "closure_sha256": "e56687506bb4a8257d2233fa90850f924b76ca7a5713bae6c85496ed017547d5", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_5f5a5b3715675e083823_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -1934,7 +1934,7 @@ "closure_sha256": "31dffe350bac3b135b79496a18fe8ff54fd2f694c13b799b6a44548ac34fe645", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_5fcc76904e5e2dd891de_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2060,7 +2060,7 @@ "closure_sha256": "1143d0e7358c89edf4c4a04a7bc8e78432e6e7d09e7a185a5e1c7e9dbd850128", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_706240be56d4e41df1a2_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2186,7 +2186,7 @@ "closure_sha256": "b4e1c6c27b131d624b5523fe1bfba006afcfb32ab55eb51a88d965aa69a5aa5d", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_7eca60b36e233943dae6_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2417,7 +2417,7 @@ "closure_sha256": "786c43a57719359c1beb01485fb1d25442f9cfb4c4903688b7066f7835fc117f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_898a1eb972fbaf55789a_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2480,7 +2480,7 @@ "closure_sha256": "e6e930a4181c41abd6c0457eb1b531c7d8b63e39add6ff252b458a3bddb71908", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_8ab0929b8154c90b5eff_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2690,7 +2690,7 @@ "closure_sha256": "70e36905352ceea9458e0c5b08237451325c082180dbf17c8c536394c0505343", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_8f3f4fb30feec66092de_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2816,7 +2816,7 @@ "closure_sha256": "f8e6586074e0ba7cc1d36c1c5d5e5df11db9060580bbd1c448ed9e52d2ecbc11", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_909a0aa57efb3f353972_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -2942,7 +2942,7 @@ "closure_sha256": "d8ebfc1b6c7b3f6dda89a22b236fe6b80a2d626c1691c8d0b7d8dc05d1d35dbb", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_945a87fa84a265fc9f94_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3005,7 +3005,7 @@ "closure_sha256": "a285ec2632103aec9553371ef3384df0e7aa2d7e443e1863bfba3af63d5b1434", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_94dcf92dabd8cc421b8f_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3152,7 +3152,7 @@ "closure_sha256": "0912ef8a3f3eefd6fcf2b9a22d00534cba2495f4543249c15164c69eda545ccd", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_9931cdff5dfd86c687c7_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3215,7 +3215,7 @@ "closure_sha256": "4e72179e078746bee49d2a82cc8e9fc1ec4c536ca912d8e20ee729e602c34c4e", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_a557058a8aeca49a6402_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3299,7 +3299,7 @@ "closure_sha256": "b07f5f119b0f0c671e1c9f130dddb4d6171b9fe7ea4886ba0a845f20c6078b21", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_ac974e171d56fc3ce10b_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -3782,7 +3782,7 @@ "closure_sha256": "d14da6eef2ff3e91ddf879047dd08ff5aad9c072b3c530e071002b4f41491ac9", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_d32455627b87ba0a1d87_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4013,7 +4013,7 @@ "closure_sha256": "c6d5dd2df0f40e42e48a1d562ceba551a678682d23f8fab8c5d0c73021fd0e4b", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_dec885b973bd44f88416_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4265,7 +4265,7 @@ "closure_sha256": "18945915ee030fd9a795214e8ddfe221ee8576f3cbe431ff79e7f965a14f8845", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_e60e7ca5d88192193e61_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4412,7 +4412,7 @@ "closure_sha256": "a674e25813771bcca4f3c050376cc2025ef0ac750fbe380faac63b461b6f9be5", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_eac26fffb3df8780302c_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4517,7 +4517,7 @@ "closure_sha256": "a2cf1b4d76a4e690725a5dbd3634cdce9600d9181866ab2a351ea0c8c26569b2", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_f011cd9add8f3c7c3eee_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4664,7 +4664,7 @@ "closure_sha256": "6282925a4476ddb4cb2d5f06faca55010bcd672651866c159766d23e0b84ad97", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_f6a70fb0288a508f2828_sm_100a", "nvcc_flags": [ "--use_fast_math" @@ -4727,7 +4727,7 @@ "closure_sha256": "5af2fdaec13f101c0636be4d72f29563d75593a3fe25587edf7994fec1b19779", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", + "min_cuda": "12.9", "name": "cake_kda_train_f92b256cccc2c9007554_sm_100a", "nvcc_flags": [ "--use_fast_math"